From 60defc5dbde98bd2ee7daa09c9e3562d1c8703b7 Mon Sep 17 00:00:00 2001 From: claracruz Date: Mon, 6 Feb 2023 16:26:51 +0000 Subject: [PATCH 001/134] Update ECMA copy (#150197) Fixes: #149159 Fixes: #150244 This PR applies ECMA copy improvements --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/move_data/move_data.test.tsx | 2 +- .../components/move_data/move_data.tsx | 18 +++++++----------- .../public/application/components/app.tsx | 18 +++++++++--------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/plugins/home/public/application/components/move_data/move_data.test.tsx b/src/plugins/home/public/application/components/move_data/move_data.test.tsx index 9c92afbf01ac3..7f8e3a1bfcf7a 100644 --- a/src/plugins/home/public/application/components/move_data/move_data.test.tsx +++ b/src/plugins/home/public/application/components/move_data/move_data.test.tsx @@ -17,6 +17,6 @@ describe('MoveData', () => { const component = shallowWithIntl(); const $button = component.find('EuiButton'); - expect($button.props().href).toBe('/app/management/data/migrate_data'); + expect($button.props().href).toBe('https://ela.st/cloud-migration'); }); }); diff --git a/src/plugins/home/public/application/components/move_data/move_data.tsx b/src/plugins/home/public/application/components/move_data/move_data.tsx index ce97e71a6ecd8..c3e0181aa4f64 100644 --- a/src/plugins/home/public/application/components/move_data/move_data.tsx +++ b/src/plugins/home/public/application/components/move_data/move_data.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import React, { FC, MouseEvent } from 'react'; +import React, { FC } from 'react'; import { EuiButton, EuiFlexGroup, @@ -19,18 +19,17 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; -import { createAppNavigationHandler } from '../app_navigation_handler'; interface Props { addBasePath: (path: string) => string; } export const MoveData: FC = ({ addBasePath }) => { - const migrateDataUrl = '/app/management/data/migrate_data'; + const migrateDataUrl = 'https://ela.st/cloud-migration'; const buttonLabel = ( ); @@ -50,7 +49,7 @@ export const MoveData: FC = ({ addBasePath }) => {

@@ -58,18 +57,15 @@ export const MoveData: FC = ({ addBasePath }) => { - {/* eslint-disable-next-line @elastic/eui/href-or-on-click */} { - createAppNavigationHandler(migrateDataUrl)(event); - }} + href={migrateDataUrl} + target="_blank" > {buttonLabel} diff --git a/x-pack/plugins/cloud_integrations/cloud_data_migration/public/application/components/app.tsx b/x-pack/plugins/cloud_integrations/cloud_data_migration/public/application/components/app.tsx index d48cb672e05c3..2823a44c09a23 100755 --- a/x-pack/plugins/cloud_integrations/cloud_data_migration/public/application/components/app.tsx +++ b/x-pack/plugins/cloud_integrations/cloud_data_migration/public/application/components/app.tsx @@ -72,7 +72,7 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr

@@ -86,7 +86,7 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr } @@ -97,8 +97,8 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr label={ } @@ -109,8 +109,8 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr label={ } @@ -121,8 +121,8 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr label={ } @@ -141,7 +141,7 @@ export const CloudDataMigrationApp = ({ http, breadcrumbService }: CloudDataMigr > From 45e57768daadd91d8757f5398cd7070932e9f941 Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Mon, 6 Feb 2023 17:44:09 +0100 Subject: [PATCH 002/134] [Infrastructure UI] Inventory educate about show button (#149981) Closes [#149615](https://github.com/elastic/kibana/issues/149615) ## Summary This PR adds a tooltip educating the user about the filter in the Show drop-down list. Testing - Navigate to the Inventory Page and the new tour should be visible: image - if it's dismissed (after the button is clicked it should not be visible anymore until the session is active - can be tested with page refresh or navigating to a different page and back) - If it's not dismissed it should be persisted per user/browser session - can be also checked with page refresh --- .../components/dropdown_button.tsx | 38 ++++++++---- .../components/kubernetes_tour.tsx | 60 +++++++++++++++++++ .../waffle/waffle_inventory_switcher.tsx | 1 + .../test/functional/apps/infra/home_page.ts | 29 +++++++-- .../test/functional/apps/infra/hosts_view.ts | 2 + .../page_objects/infra_home_page.ts | 14 +++++ 6 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/infra/public/pages/metrics/inventory_view/components/kubernetes_tour.tsx diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/dropdown_button.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/dropdown_button.tsx index 89bc192606944..0e4aa275a7879 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/dropdown_button.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/dropdown_button.tsx @@ -8,6 +8,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; import React, { ReactNode } from 'react'; import { withTheme, EuiTheme } from '@kbn/kibana-react-plugin/common'; +import { KubernetesTour } from './kubernetes_tour'; interface Props { 'data-test-subj'?: string; @@ -15,10 +16,26 @@ interface Props { onClick: () => void; theme: EuiTheme | undefined; children: ReactNode; + showKubernetesInfo?: boolean; } +const ButtonLabel = ({ label, theme }: { label: string; theme?: EuiTheme }) => ( + + {label} + +); + export const DropdownButton = withTheme((props: Props) => { - const { onClick, label, theme, children } = props; + const { onClick, label, theme, children, showKubernetesInfo } = props; return ( { boxShadow: `0px 3px 2px ${theme?.eui.euiTableActionsBorderColor}, 0px 1px 1px ${theme?.eui.euiTableActionsBorderColor}`, }} > - - {label} - + {showKubernetesInfo ? ( + + + + ) : ( + + )} { + const [isTourSeen, setIsTourSeen] = useLocalStorage(KUBERNETES_TOUR_STORAGE_KEY, false); + const markTourAsSeen = () => setIsTourSeen(true); + + return ( +
+ + {i18n.translate('xpack.infra.homePage.kubernetesTour.text', { + defaultMessage: + 'Click here to see your infrastructure in different ways, including Kubernetes pods.', + })} + + } + isStepOpen={!isTourSeen} + maxWidth={350} + onFinish={markTourAsSeen} + step={1} + stepsTotal={1} + title={i18n.translate('xpack.infra.homePage.kubernetesTour.title', { + defaultMessage: 'Want a different view?', + })} + anchorPosition="downCenter" + footerAction={ + + {i18n.translate('xpack.infra.homePage.kubernetesTour.dismiss', { + defaultMessage: 'Dismiss', + })} + + } + > + {children} + +
+ ); +}; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/waffle_inventory_switcher.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/waffle_inventory_switcher.tsx index 6234dcc0bbf0b..e3ca9b770e2ef 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/waffle_inventory_switcher.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/waffle/waffle_inventory_switcher.tsx @@ -124,6 +124,7 @@ export const WaffleInventorySwitcher: React.FC = () => { data-test-subj={'openInventorySwitcher'} onClick={openPopover} label={i18n.translate('xpack.infra.waffle.showLabel', { defaultMessage: 'Show' })} + showKubernetesInfo={true} > {selectedText} diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts index 1c0323808270d..54435f11a33e1 100644 --- a/x-pack/test/functional/apps/infra/home_page.ts +++ b/x-pack/test/functional/apps/infra/home_page.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { KUBERNETES_TOUR_STORAGE_KEY } from '@kbn/infra-plugin/public/pages/metrics/inventory_view/components/kubernetes_tour'; import { FtrProviderContext } from '../../ftr_provider_context'; import { DATES } from './constants'; @@ -59,10 +60,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await pageObjects.common.navigateToApp('infraOps'); await pageObjects.infraHome.waitForLoading(); }); - after( - async () => - await esArchiver.unload('x-pack/test/functional/es_archives/infra/metrics_and_logs') - ); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/infra/metrics_and_logs'); + await browser.removeLocalStorageItem(KUBERNETES_TOUR_STORAGE_KEY); + }); it('renders the correct page title', async () => { await pageObjects.header.waitUntilLoadingHasFinished(); @@ -71,6 +72,26 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { expect(documentTitle).to.contain('Inventory - Infrastructure - Observability - Elastic'); }); + it('renders the kubernetes tour component and allows user to dismiss it without seeing it again', async () => { + await pageObjects.header.waitUntilLoadingHasFinished(); + const kubernetesTourText = + 'Click here to see your infrastructure in different ways, including Kubernetes pods.'; + const ensureKubernetesTourVisible = + await pageObjects.infraHome.ensureKubernetesTourIsVisible(); + + expect(ensureKubernetesTourVisible).to.contain(kubernetesTourText); + + // Persist after refresh + await browser.refresh(); + await pageObjects.infraHome.waitForLoading(); + + expect(ensureKubernetesTourVisible).to.contain(kubernetesTourText); + + await pageObjects.infraHome.clickDismissKubernetesTourButton(); + + await pageObjects.infraHome.ensureKubernetesTourIsClosed(); + }); + it('renders an empty data prompt for dates with no data', async () => { await pageObjects.infraHome.goToTime(DATE_WITHOUT_DATA); await pageObjects.infraHome.getNoMetricsDataPrompt(); diff --git a/x-pack/test/functional/apps/infra/hosts_view.ts b/x-pack/test/functional/apps/infra/hosts_view.ts index 9975a2772a193..224a924b7c1fa 100644 --- a/x-pack/test/functional/apps/infra/hosts_view.ts +++ b/x-pack/test/functional/apps/infra/hosts_view.ts @@ -108,6 +108,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs'); await pageObjects.common.navigateToApp('infraOps'); + await pageObjects.infraHome.clickDismissKubernetesTourButton(); await pageObjects.infraHostsView.clickTryHostViewBadge(); }); after(async () => { @@ -126,6 +127,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs'); await loginWithReadOnlyUserAndNavigateToInfra(); + await pageObjects.infraHome.clickDismissKubernetesTourButton(); await pageObjects.infraHostsView.clickTryHostViewBadge(); }); after(async () => { diff --git a/x-pack/test/functional/page_objects/infra_home_page.ts b/x-pack/test/functional/page_objects/infra_home_page.ts index cb01de3c596e3..a13c9a89ae0cb 100644 --- a/x-pack/test/functional/page_objects/infra_home_page.ts +++ b/x-pack/test/functional/page_objects/infra_home_page.ts @@ -376,5 +376,19 @@ export function InfraHomePageProvider({ getService, getPageObjects }: FtrProvide async ensureSuggestionsPanelVisible() { await testSubjects.find('infraSuggestionsPanel'); }, + + async ensureKubernetesTourIsVisible() { + const container = await testSubjects.find('infra-kubernetesTour-text'); + const containerText = await container.getVisibleText(); + return containerText; + }, + + async ensureKubernetesTourIsClosed() { + await testSubjects.missingOrFail('infra-kubernetesTour-text'); + }, + + async clickDismissKubernetesTourButton() { + return await testSubjects.click('infra-kubernetesTour-dismiss'); + }, }; } From 8733774e8137fb531d4531fad58e09b65bba8689 Mon Sep 17 00:00:00 2001 From: Wafaa Nasr Date: Mon, 6 Feb 2023 16:47:22 +0000 Subject: [PATCH 003/134] [Security Solution][Platform] - Add connectors to import/export API (#148703) ## Summary - [x] Addresses https://github.com/elastic/kibana/issues/118774 - [x] Enable Security Rule to be **imported** even if one of its connectors has a missing secret - [x] Shows **Warning Callout** in the Import Modal when missing secrets connector is imported. - [x] Added Link `connectors` to the connectors page in the same tab, so that the user can fix imported connectors. - [x] Added `Overwrite existing connectors with conflicting action "id"` option to the Import Modal ## Cases: > **Export:** > - Export Rule(s) with connectors through `Export All` or `Bulk Actions` > > **Import:** > - Import Rule with correct connectors data > - Import Rule with missing secrets' connectors by showing a warning callout > - Re-Import connectors even if they were stored when overwrite is true > > **Error:** > - Showing an error message when the user has a Read Actions permission and stops the importing => ` You may not have actions privileges required to import rules with actions ...` > - Showing an error message when the user has an old imported rule missing all the connectors data **OR** these connectors were not in the user's env => `X connector is missing. Connector id missing is: X` > - Showing an error if the new connectors defined in the exported file are not corresponding to the actions array under the rules param => `X connector is missing. Connector id missing is: X` > - **Showing a ` conflict` error in case of existing connectors and re-importing again with an `overwrite` false => this won't happen in case of implementing the `Skipping action-connectors importing if all connectors have been imported/created before`** > > **Skip importing:** > - Skipping action-connectors importing if the `actions` array is empty, even if the user has exported-connectors in the file > - Skipping action-connectors importing if all connectors have been imported/created before > ### Screenshots > > **1. Importing Connectors successfully** > image > > **2. Importing Connectors with warnings** image **3.Connector Page** image ## New text: @nastasha-solomon **1. Warning message** title => could be ` 1 connector imported` or `x connectors imported` message => ` 1 connector has sensitive information that requires updates. review in connectors` or `x connectors have sensitive information that requires updates. review in connectors` image **2. New `Overwrite` checkbox** image **3. Success Toast message** image **4. Error messages** a. Missing import action privileges image b. Missing connectors image image - [x] References: Use **getImporter** and **getExporter** from Saved Object [Connectors SO import/export implementation](https://github.com/elastic/kibana/pull/98802) , [Kibana-Core confirmation](https://elastic.slack.com/archives/C5TQ33ND8/p1673275186013589 ) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../src/import_query_schema/index.test.ts | 21 + .../src/import_query_schema/index.ts | 4 +- .../rules_client/lib/validate_actions.ts | 35 +- .../server/rules_client/methods/create.ts | 8 +- .../server/rules_client/methods/update.ts | 17 +- .../server/rules_client/tests/create.test.ts | 116 +++ .../server/rules_client/tests/update.test.ts | 149 ++++ .../import_rules/response_schema.test.ts | 222 ++++++ .../api/rules/import_rules/response_schema.ts | 6 +- .../export_rules_details_schema.mock.ts | 19 + .../export_rules_details_schema.test.ts | 20 +- .../export/export_rules_details_schema.ts | 46 +- .../model/import/rule_to_import.mock.ts | 38 + .../schemas/response/index.ts | 1 + .../schemas/response/warning_schema/index.ts | 24 + .../security_solution/cypress/objects/rule.ts | 5 + .../cypress/screens/alerts_detection_rules.ts | 7 +- .../cypress/tasks/alerts_detection_rules.ts | 8 +- .../__snapshots__/index.test.tsx.snap | 514 ++++++++++++- .../action_connectors_warning.test.tsx.snap | 678 ++++++++++++++++++ .../action_connectors_warning.test.tsx | 104 +++ .../action_connectors_warning/index.tsx | 65 ++ .../import_data_modal/index.test.tsx | 114 ++- .../components/import_data_modal/index.tsx | 77 +- .../import_data_modal/translations.ts | 49 +- .../import_data_modal/utils.test.ts | 291 +++++++- .../components/import_data_modal/utils.ts | 94 ++- .../rule_management/api/api.test.ts | 10 + .../rule_management/api/api.ts | 7 +- .../rule_management/logic/types.ts | 6 + .../pages/rule_management/index.tsx | 1 + .../api/rules/bulk_actions/route.ts | 12 +- .../api/rules/export_rules/route.ts | 27 +- .../api/rules/import_rules/route.test.ts | 48 ++ .../api/rules/import_rules/route.ts | 46 +- .../logic/crud/create_rules.ts | 3 + .../rule_management/logic/crud/patch_rules.ts | 3 + .../logic/export/get_export_all.test.ts | 191 ++++- .../logic/export/get_export_all.ts | 20 +- .../export/get_export_by_object_ids.test.ts | 193 ++++- .../logic/export/get_export_by_object_ids.ts | 29 +- .../logic/export/get_export_details_ndjson.ts | 20 +- .../get_export_rule_action_connectors.ts | 92 +++ .../import_rule_action_connectors.test.ts | 383 ++++++++++ .../import_rule_action_connectors.ts | 84 +++ .../logic/import/action_connectors/types.ts | 44 ++ .../import/action_connectors/utils/index.ts | 129 ++++ .../import/create_rules_stream_from_ndjson.ts | 8 + .../logic/import/import_rules_utils.ts | 7 +- .../plugins/security_solution/tsconfig.json | 2 + .../basic/tests/export_rules.ts | 5 + .../basic/tests/import_rules.ts | 36 + .../group1/export_rules.ts | 5 + .../group10/import_rules.ts | 334 ++++++++- .../group10/perform_bulk_action.ts | 94 +++ .../security_and_spaces/tests/import_rules.ts | 86 ++- 56 files changed, 4475 insertions(+), 182 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/schemas/response/warning_schema/index.ts create mode 100644 x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/__snapshots__/action_connectors_warning.test.tsx.snap create mode 100644 x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/action_connectors_warning.test.tsx create mode 100644 x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/index.tsx create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_rule_action_connectors.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/utils/index.ts diff --git a/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.test.ts b/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.test.ts index d2f84b4ee3708..ab1f35c54b5db 100644 --- a/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.test.ts +++ b/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.test.ts @@ -16,6 +16,7 @@ describe('importQuerySchema', () => { as_new_list: false, overwrite: true, overwrite_exceptions: true, + overwrite_action_connectors: true, }; const decoded = importQuerySchema.decode(payload); const checked = exactCheck(payload, decoded); @@ -30,6 +31,7 @@ describe('importQuerySchema', () => { as_new_list: false, overwrite: 'wrong', overwrite_exceptions: true, + overwrite_action_connectors: true, }; const decoded = importQuerySchema.decode(payload); const checked = exactCheck(payload, decoded); @@ -48,6 +50,7 @@ describe('importQuerySchema', () => { as_new_list: false, overwrite: true, overwrite_exceptions: 'wrong', + overwrite_action_connectors: true, }; const decoded = importQuerySchema.decode(payload); const checked = exactCheck(payload, decoded); @@ -58,6 +61,24 @@ describe('importQuerySchema', () => { ]); expect(message.schema).toEqual({}); }); + test('it should NOT validate a non boolean value for "overwrite_action_connectors"', () => { + const payload: Omit & { + overwrite_action_connectors: string; + } = { + as_new_list: false, + overwrite: true, + overwrite_exceptions: true, + overwrite_action_connectors: 'wrong', + }; + const decoded = importQuerySchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = foldLeftRight(checked); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "wrong" supplied to "overwrite_action_connectors"', + ]); + expect(message.schema).toEqual({}); + }); test('it should NOT allow an extra key to be sent in', () => { const payload: ImportQuerySchema & { diff --git a/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.ts b/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.ts index ca0e35c4aa176..5b1ee38bb0855 100644 --- a/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-types/src/import_query_schema/index.ts @@ -14,6 +14,7 @@ export const importQuerySchema = t.exact( t.partial({ overwrite: DefaultStringBooleanFalse, overwrite_exceptions: DefaultStringBooleanFalse, + overwrite_action_connectors: DefaultStringBooleanFalse, as_new_list: DefaultStringBooleanFalse, }) ); @@ -21,9 +22,10 @@ export const importQuerySchema = t.exact( export type ImportQuerySchema = t.TypeOf; export type ImportQuerySchemaDecoded = Omit< ImportQuerySchema, - 'overwrite' | 'overwrite_exceptions' | 'as_new_list' + 'overwrite' | 'overwrite_exceptions' | 'as_new_list' | 'overwrite_action_connectors' > & { overwrite: boolean; overwrite_exceptions: boolean; + overwrite_action_connectors: boolean; as_new_list: boolean; }; diff --git a/x-pack/plugins/alerting/server/rules_client/lib/validate_actions.ts b/x-pack/plugins/alerting/server/rules_client/lib/validate_actions.ts index 3c97080e2d655..ea6145b8f6e99 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/validate_actions.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/validate_actions.ts @@ -17,7 +17,10 @@ import { parseDuration } from '../../lib'; export async function validateActions( context: RulesClientContext, alertType: UntypedNormalizedRuleType, - data: Pick & { actions: NormalizedAlertAction[] } + data: Pick & { + actions: NormalizedAlertAction[]; + }, + allowMissingConnectorSecrets?: boolean ): Promise { const { actions, notifyWhen, throttle } = data; const hasRuleLevelNotifyWhen = typeof notifyWhen !== 'undefined'; @@ -35,20 +38,26 @@ export async function validateActions( const actionsUsingConnectorsWithMissingSecrets = actionResults.filter( (result) => result.isMissingSecrets ); - if (actionsUsingConnectorsWithMissingSecrets.length) { - errors.push( - i18n.translate('xpack.alerting.rulesClient.validateActions.misconfiguredConnector', { - defaultMessage: 'Invalid connectors: {groups}', - values: { - groups: actionsUsingConnectorsWithMissingSecrets - .map((connector) => connector.name) - .join(', '), - }, - }) - ); + if (allowMissingConnectorSecrets) { + context.logger.error( + `Invalid connectors with "allowMissingConnectorSecrets": ${actionsUsingConnectorsWithMissingSecrets + .map((connector) => connector.name) + .join(', ')}` + ); + } else { + errors.push( + i18n.translate('xpack.alerting.rulesClient.validateActions.misconfiguredConnector', { + defaultMessage: 'Invalid connectors: {groups}', + values: { + groups: actionsUsingConnectorsWithMissingSecrets + .map((connector) => connector.name) + .join(', '), + }, + }) + ); + } } - // check for actions with invalid action groups const { actionGroups: alertTypeActionGroups } = alertType; const usedAlertActionGroups = actions.map((action) => action.group); diff --git a/x-pack/plugins/alerting/server/rules_client/methods/create.ts b/x-pack/plugins/alerting/server/rules_client/methods/create.ts index e8dcefe4a9ef1..09dc8f62494b8 100644 --- a/x-pack/plugins/alerting/server/rules_client/methods/create.ts +++ b/x-pack/plugins/alerting/server/rules_client/methods/create.ts @@ -46,11 +46,12 @@ export interface CreateOptions { | 'nextRun' > & { actions: NormalizedAlertAction[] }; options?: SavedObjectOptions; + allowMissingConnectorSecrets?: boolean; } export async function create( context: RulesClientContext, - { data, options }: CreateOptions + { data, options, allowMissingConnectorSecrets }: CreateOptions ): Promise> { const id = options?.id || SavedObjectsUtils.generateId(); @@ -104,10 +105,11 @@ export async function create( } } - await validateActions(context, ruleType, data); + await validateActions(context, ruleType, data, allowMissingConnectorSecrets); await withSpan({ name: 'validateActions', type: 'rules' }, () => - validateActions(context, ruleType, data) + validateActions(context, ruleType, data, allowMissingConnectorSecrets) ); + // Throw error if schedule interval is less than the minimum and we are enforcing it const intervalInMs = parseDuration(data.schedule.interval); if ( diff --git a/x-pack/plugins/alerting/server/rules_client/methods/update.ts b/x-pack/plugins/alerting/server/rules_client/methods/update.ts index 8fa3855b95707..5e116f9f21b28 100644 --- a/x-pack/plugins/alerting/server/rules_client/methods/update.ts +++ b/x-pack/plugins/alerting/server/rules_client/methods/update.ts @@ -38,22 +38,23 @@ export interface UpdateOptions { throttle?: string | null; notifyWhen?: RuleNotifyWhenType | null; }; + allowMissingConnectorSecrets?: boolean; } export async function update( context: RulesClientContext, - { id, data }: UpdateOptions + { id, data, allowMissingConnectorSecrets }: UpdateOptions ): Promise> { return await retryIfConflicts( context.logger, `rulesClient.update('${id}')`, - async () => await updateWithOCC(context, { id, data }) + async () => await updateWithOCC(context, { id, data, allowMissingConnectorSecrets }) ); } async function updateWithOCC( context: RulesClientContext, - { id, data }: UpdateOptions + { id, data, allowMissingConnectorSecrets }: UpdateOptions ): Promise> { let alertSavedObject: SavedObject; @@ -99,7 +100,11 @@ async function updateWithOCC( context.ruleTypeRegistry.ensureRuleTypeEnabled(alertSavedObject.attributes.alertTypeId); - const updateResult = await updateAlert(context, { id, data }, alertSavedObject); + const updateResult = await updateAlert( + context, + { id, data, allowMissingConnectorSecrets }, + alertSavedObject + ); await Promise.all([ alertSavedObject.attributes.apiKey @@ -138,7 +143,7 @@ async function updateWithOCC( async function updateAlert( context: RulesClientContext, - { id, data }: UpdateOptions, + { id, data, allowMissingConnectorSecrets }: UpdateOptions, { attributes, version }: SavedObject ): Promise> { const ruleType = context.ruleTypeRegistry.get(attributes.alertTypeId); @@ -156,7 +161,7 @@ async function updateAlert( // Validate const validatedAlertTypeParams = validateRuleTypeParams(data.params, ruleType.validate?.params); - await validateActions(context, ruleType, data); + await validateActions(context, ruleType, data, allowMissingConnectorSecrets); // Throw error if schedule interval is less than the minimum and we are enforcing it const intervalInMs = parseDuration(data.schedule.interval); diff --git a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts index c11dc8be21ca4..dddfbe85f9ee3 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/create.test.ts @@ -3047,4 +3047,120 @@ describe('create()', () => { expect(unsecuredSavedObjectsClient.create).not.toHaveBeenCalled(); expect(taskManager.schedule).not.toHaveBeenCalled(); }); + test('should create a rule even if action is missing secret when allowMissingConnectorSecrets is true', async () => { + const data = getMockData({ + actions: [ + { + group: 'default', + id: '1', + params: { + foo: true, + }, + }, + { + group: 'default', + id: '1', + params: { + foo: true, + }, + }, + { + group: 'default', + id: '2', + params: { + foo: true, + }, + }, + ], + }); + actionsClient.getBulk.mockReset(); + actionsClient.getBulk.mockResolvedValue([ + { + id: '1', + actionTypeId: '.slack', + config: {}, + isMissingSecrets: true, + name: 'Slack connector', + isPreconfigured: false, + isDeprecated: false, + }, + ]); + unsecuredSavedObjectsClient.create.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + alertTypeId: '123', + schedule: { interval: '1m' }, + params: { + bar: true, + }, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + notifyWhen: null, + actions: [ + { + group: 'default', + actionRef: 'action_0', + actionTypeId: '.slack', + params: { + foo: true, + }, + }, + ], + }, + references: [ + { + name: 'action_0', + type: 'action', + id: '1', + }, + { + name: 'action_1', + type: 'action', + id: '1', + }, + { + name: 'action_2', + type: 'action', + id: '2', + }, + ], + }); + unsecuredSavedObjectsClient.create.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + actions: [], + scheduledTaskId: 'task-123', + }, + references: [], + }); + const result = await rulesClient.create({ data, allowMissingConnectorSecrets: true }); + expect(result).toMatchInlineSnapshot(` + Object { + "actions": Array [ + Object { + "actionTypeId": ".slack", + "group": "default", + "id": "1", + "params": Object { + "foo": true, + }, + }, + ], + "alertTypeId": "123", + "createdAt": 2019-02-12T21:01:22.479Z, + "id": "1", + "notifyWhen": null, + "params": Object { + "bar": true, + }, + "schedule": Object { + "interval": "1m", + }, + "scheduledTaskId": "task-123", + "updatedAt": 2019-02-12T21:01:22.479Z, + } + `); + }); }); diff --git a/x-pack/plugins/alerting/server/rules_client/tests/update.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/update.test.ts index 44c4eeb50fe27..4d16f3e5d66a0 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/update.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/update.test.ts @@ -2022,6 +2022,155 @@ describe('update()', () => { expect(unsecuredSavedObjectsClient.create).not.toHaveBeenCalled(); expect(taskManager.schedule).not.toHaveBeenCalled(); }); + test('should update a rule even if action is missing secret when allowMissingConnectorSecrets is true', async () => { + // Reset from default behaviour + actionsClient.getBulk.mockReset(); + actionsClient.getBulk.mockResolvedValue([ + { + id: '1', + actionTypeId: '.slack', + config: {}, + isMissingSecrets: true, + name: 'slack connector', + isPreconfigured: false, + isDeprecated: false, + }, + ]); + actionsClient.isPreconfigured.mockReset(); + actionsClient.isPreconfigured.mockReturnValueOnce(false); + actionsClient.isPreconfigured.mockReturnValueOnce(true); + actionsClient.isPreconfigured.mockReturnValueOnce(true); + unsecuredSavedObjectsClient.create.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + enabled: true, + schedule: { interval: '1m' }, + params: { + bar: true, + }, + actions: [ + { + group: 'default', + actionRef: 'action_0', + actionTypeId: '.slack', + params: { + foo: true, + }, + }, + ], + notifyWhen: 'onActiveAlert', + scheduledTaskId: 'task-123', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + references: [ + { + name: 'action_0', + type: 'action', + id: '1', + }, + { + name: 'param:soRef_0', + type: 'someSavedObjectType', + id: '9', + }, + ], + }); + const result = await rulesClient.update({ + id: '1', + data: { + schedule: { interval: '1m' }, + name: 'abc', + tags: ['foo'], + params: { + bar: true, + }, + throttle: null, + notifyWhen: 'onActiveAlert', + actions: [ + { + group: 'default', + id: '1', + params: { + foo: true, + }, + }, + ], + }, + allowMissingConnectorSecrets: true, + }); + + expect(unsecuredSavedObjectsClient.create).toHaveBeenNthCalledWith( + 1, + 'alert', + { + actions: [ + { + group: 'default', + actionRef: 'action_0', + actionTypeId: '.slack', + params: { + foo: true, + }, + }, + ], + alertTypeId: 'myType', + apiKey: null, + apiKeyOwner: null, + consumer: 'myApp', + enabled: true, + meta: { versionApiKeyLastmodified: 'v7.10.0' }, + name: 'abc', + notifyWhen: 'onActiveAlert', + params: { bar: true }, + schedule: { interval: '1m' }, + scheduledTaskId: 'task-123', + tags: ['foo'], + throttle: null, + updatedAt: '2019-02-12T21:01:22.479Z', + updatedBy: 'elastic', + }, + { + id: '1', + overwrite: true, + references: [{ id: '1', name: 'action_0', type: 'action' }], + version: '123', + } + ); + + expect(result).toMatchInlineSnapshot(` + Object { + "actions": Array [ + Object { + "actionTypeId": ".slack", + "group": "default", + "id": "1", + "params": Object { + "foo": true, + }, + }, + ], + "createdAt": 2019-02-12T21:01:22.479Z, + "enabled": true, + "id": "1", + "notifyWhen": "onActiveAlert", + "params": Object { + "bar": true, + }, + "schedule": Object { + "interval": "1m", + }, + "scheduledTaskId": "task-123", + "updatedAt": 2019-02-12T21:01:22.479Z, + } + `); + expect(encryptedSavedObjects.getDecryptedAsInternalUser).toHaveBeenCalledWith('alert', '1', { + namespace: 'default', + }); + expect(unsecuredSavedObjectsClient.get).not.toHaveBeenCalled(); + expect(actionsClient.isPreconfigured).toHaveBeenCalledTimes(1); + }); test('logs warning when creating with an interval less than the minimum configured one when enforce = false', async () => { actionsClient.getBulk.mockReset(); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.test.ts index 2f11e1a9c120f..b2ff257ba2c4e 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.test.ts @@ -24,6 +24,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -42,6 +46,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -60,6 +68,10 @@ describe('Import rules response schema', () => { exceptions_errors: [{ error: { status_code: 400, message: 'some message' } }], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -81,6 +93,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -102,6 +118,10 @@ describe('Import rules response schema', () => { ], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -120,6 +140,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -140,6 +164,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: -1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -178,6 +206,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded as UnsafeCastForTest); @@ -217,6 +249,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: 'hello', exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded as UnsafeCastForTest); @@ -238,6 +274,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -262,6 +302,10 @@ describe('Import rules response schema', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }; const decoded = ImportRulesResponse.decode(payload); const checked = exactCheck(payload, decoded); @@ -270,4 +314,182 @@ describe('Import rules response schema', () => { expect(getPaths(left(message.errors))).toEqual(['invalid keys "invalid_data"']); expect(message.schema).toEqual({}); }); + + test('it should validate an empty import response with a single connectors error', () => { + const payload: ImportRulesResponse = { + success: false, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [{ error: { status_code: 400, message: 'some message' } }], + action_connectors_warnings: [], + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should validate an empty import response with multiple errors', () => { + const payload: ImportRulesResponse = { + success: false, + success_count: 0, + rules_count: 0, + errors: [ + { error: { status_code: 400, message: 'some message' } }, + { error: { status_code: 500, message: 'some message' } }, + ], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [{ error: { status_code: 400, message: 'some message' } }], + action_connectors_warnings: [], + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should NOT validate action_connectors_success that is not boolean', () => { + type UnsafeCastForTest = Either< + Errors, + { + success: boolean; + action_connectors_success: string; + success_count: number; + errors: Array< + { + id?: string | undefined; + rule_id?: string | undefined; + } & { + error: { + status_code: number; + message: string; + }; + } + >; + } + >; + const payload: Omit & { + action_connectors_success: string; + } = { + success: true, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: 'invalid', + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded as UnsafeCastForTest); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "invalid" supplied to "action_connectors_success"', + ]); + expect(message.schema).toEqual({}); + }); + test('it should NOT validate a action_connectors_success_count that is a negative number', () => { + const payload: ImportRulesResponse = { + success: false, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: -1, + action_connectors_errors: [], + action_connectors_warnings: [], + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "action_connectors_success_count"', + ]); + expect(message.schema).toEqual({}); + }); + test('it should validate a action_connectors_warnings after importing successfully', () => { + const payload: ImportRulesResponse = { + success: false, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 1, + action_connectors_errors: [], + action_connectors_warnings: [{ type: 'type', message: 'message', actionPath: 'actionPath' }], + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + test('it should NOT validate a action_connectors_warnings that is not WarningSchema', () => { + type UnsafeCastForTest = Either< + Errors, + { + success: boolean; + action_connectors_warnings: string; + success_count: number; + errors: Array< + { + id?: string | undefined; + rule_id?: string | undefined; + } & { + error: { + status_code: number; + message: string; + }; + } + >; + } + >; + const payload: Omit & { + action_connectors_warnings: string; + } = { + success: true, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: 'invalid', + }; + const decoded = ImportRulesResponse.decode(payload); + const checked = exactCheck(payload, decoded as UnsafeCastForTest); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "invalid" supplied to "action_connectors_warnings"', + ]); + expect(message.schema).toEqual({}); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.ts index 77ccd0812c2c9..99212b902c4d3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/api/rules/import_rules/response_schema.ts @@ -7,7 +7,7 @@ import * as t from 'io-ts'; import { PositiveInteger } from '@kbn/securitysolution-io-ts-types'; -import { errorSchema } from '../../../../schemas/response/error_schema'; +import { errorSchema, warningSchema } from '../../../../schemas/response'; export type ImportRulesResponse = t.TypeOf; export const ImportRulesResponse = t.exact( @@ -19,5 +19,9 @@ export const ImportRulesResponse = t.exact( success: t.boolean, success_count: PositiveInteger, errors: t.array(errorSchema), + action_connectors_errors: t.array(errorSchema), + action_connectors_warnings: t.array(warningSchema), + action_connectors_success: t.boolean, + action_connectors_success_count: PositiveInteger, }) ); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.mock.ts index 64b82abdb3755..9a26b1a09f066 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.mock.ts @@ -8,6 +8,7 @@ import { getExceptionExportDetailsMock } from '@kbn/lists-plugin/common/schemas/response/exception_export_details_schema.mock'; import type { ExportExceptionDetailsMock } from '@kbn/lists-plugin/common/schemas/response/exception_export_details_schema.mock'; import type { ExportRulesDetails } from './export_rules_details_schema'; +import type { DefaultActionConnectorDetails } from '../../../../../server/lib/detection_engine/rule_management/logic/export/get_export_rule_action_connectors'; interface RuleDetailsMock { totalCount?: number; @@ -16,6 +17,23 @@ interface RuleDetailsMock { missingRules?: Array>; } +export const getActionConnectorDetailsMock = (): DefaultActionConnectorDetails => ({ + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], + excluded_action_connection_count: 0, + excluded_action_connections: [], +}); +export const getOutputDetailsSampleWithActionConnectors = (): ExportRulesDetails => ({ + ...getOutputDetailsSample(), + ...getExceptionExportDetailsMock(), + exported_action_connector_count: 1, + missing_action_connection_count: 0, + missing_action_connections: [], + excluded_action_connection_count: 0, + excluded_action_connections: [], +}); + export const getOutputDetailsSample = (ruleDetails?: RuleDetailsMock): ExportRulesDetails => ({ exported_count: ruleDetails?.totalCount ?? 0, exported_rules_count: ruleDetails?.rulesCount ?? 0, @@ -29,6 +47,7 @@ export const getOutputDetailsSampleWithExceptions = ( ): ExportRulesDetails => ({ ...getOutputDetailsSample(ruleDetails), ...getExceptionExportDetailsMock(exceptionDetails), + ...getActionConnectorDetailsMock(), }); export const getSampleDetailsAsNdjson = (sample: ExportRulesDetails): string => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.test.ts index 2c84a34b34928..a91f59924ef72 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.test.ts @@ -18,15 +18,16 @@ import { exactCheck, foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts import { getOutputDetailsSample, + getOutputDetailsSampleWithActionConnectors, getOutputDetailsSampleWithExceptions, } from './export_rules_details_schema.mock'; import type { ExportRulesDetails } from './export_rules_details_schema'; -import { exportRulesDetailsWithExceptionsSchema } from './export_rules_details_schema'; +import { exportRulesDetailsWithExceptionsAndConnectorsSchema } from './export_rules_details_schema'; -describe('exportRulesDetailsWithExceptionsSchema', () => { +describe('exportRulesDetailsWithExceptionsAndConnectorsSchema', () => { test('it should validate export details response', () => { const payload = getOutputDetailsSample(); - const decoded = exportRulesDetailsWithExceptionsSchema.decode(payload); + const decoded = exportRulesDetailsWithExceptionsAndConnectorsSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); @@ -36,7 +37,7 @@ describe('exportRulesDetailsWithExceptionsSchema', () => { test('it should validate export details with exceptions details response', () => { const payload = getOutputDetailsSampleWithExceptions(); - const decoded = exportRulesDetailsWithExceptionsSchema.decode(payload); + const decoded = exportRulesDetailsWithExceptionsAndConnectorsSchema.decode(payload); const checked = exactCheck(payload, decoded); const message = pipe(checked, foldLeftRight); @@ -44,12 +45,21 @@ describe('exportRulesDetailsWithExceptionsSchema', () => { expect(message.schema).toEqual(payload); }); + test('it should validate export details with action connectors details response', () => { + const payload = getOutputDetailsSampleWithActionConnectors(); + const decoded = exportRulesDetailsWithExceptionsAndConnectorsSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); test('it should strip out extra keys', () => { const payload: ExportRulesDetails & { extraKey?: string; } = getOutputDetailsSample(); payload.extraKey = 'some extra key'; - const decoded = exportRulesDetailsWithExceptionsSchema.decode(payload); + const decoded = exportRulesDetailsWithExceptionsAndConnectorsSchema.decode(payload); const message = pipe(decoded, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.ts index 85b423135566b..205861bdd42a7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/export/export_rules_details_schema.ts @@ -9,13 +9,6 @@ import * as t from 'io-ts'; import { exportExceptionDetails } from '@kbn/securitysolution-io-ts-list-types'; import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; -const createSchema = ( - requiredFields: Required, - optionalFields: Optional -) => { - return t.intersection([t.exact(t.type(requiredFields)), t.exact(t.partial(optionalFields))]); -}; - const exportRulesDetails = { exported_count: t.number, exported_rules_count: t.number, @@ -28,11 +21,38 @@ const exportRulesDetails = { ), missing_rules_count: t.number, }; +const excludedActionConnectors = t.intersection([ + t.exact( + t.type({ + id: NonEmptyString, + type: NonEmptyString, + }) + ), + t.exact(t.partial({ reason: t.string })), +]); + +const exportRuleActionConnectorsDetails = { + exported_action_connector_count: t.number, + missing_action_connection_count: t.number, + missing_action_connections: t.array( + t.exact( + t.type({ + id: NonEmptyString, + type: NonEmptyString, + }) + ) + ), + excluded_action_connection_count: t.number, + excluded_action_connections: t.array(excludedActionConnectors), +}; -// With exceptions -export const exportRulesDetailsWithExceptionsSchema = createSchema( - exportRulesDetails, - exportExceptionDetails -); +// With exceptions and connectors +export const exportRulesDetailsWithExceptionsAndConnectorsSchema = t.intersection([ + t.exact(t.type(exportRulesDetails)), + t.exact(t.partial(exportExceptionDetails)), + t.exact(t.partial(exportRuleActionConnectorsDetails)), +]); -export type ExportRulesDetails = t.TypeOf; +export type ExportRulesDetails = t.TypeOf< + typeof exportRulesDetailsWithExceptionsAndConnectorsSchema +>; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/import/rule_to_import.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/import/rule_to_import.mock.ts index d1dc9e8ac4663..cec58d1c5fdc6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/import/rule_to_import.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/model/import/rule_to_import.mock.ts @@ -80,3 +80,41 @@ export const getImportThreatMatchRulesSchemaMock = (ruleId = 'rule-1'): RuleToIm }, ], }); + +export const webHookConnector = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: false, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', +}; + +export const ruleWithConnectorNdJSON = (): string => { + const items = [ + { + ...getImportRulesSchemaMock(), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + action_type_id: '.webhook', + params: {}, + }, + ], + }, + webHookConnector, + ]; + const stringOfExceptions = items.map((item) => JSON.stringify(item)); + + return stringOfExceptions.join('\n'); +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/index.ts index b20a956525e2e..76da687604028 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/index.ts @@ -6,3 +6,4 @@ */ export * from './error_schema'; +export * from './warning_schema'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/warning_schema/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/warning_schema/index.ts new file mode 100644 index 0000000000000..1a401d1941cb0 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/warning_schema/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +const partial = t.exact( + t.partial({ + buttonLabel: t.string, + }) +); +const required = t.exact( + t.type({ + type: t.string, + message: t.string, + actionPath: t.string, + }) +); + +export const warningSchema = t.intersection([partial, required]); +export type WarningSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index 20f41f938502b..a80c7099c31ff 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -573,6 +573,11 @@ export const expectedExportedRule = (ruleResponse: Cypress.Response { .pipe(($el) => $el.trigger('click')) .should('be.checked'); }; - +const selectOverwriteConnectorsRulesImport = () => { + cy.get(RULE_IMPORT_OVERWRITE_CONNECTORS_CHECKBOX) + .pipe(($el) => $el.trigger('click')) + .should('be.checked'); +}; export const importRulesWithOverwriteAll = (rulesFile: string) => { cy.get(RULE_IMPORT_MODAL).click(); cy.get(INPUT_FILE).should('exist'); cy.get(INPUT_FILE).trigger('click', { force: true }).attachFile(rulesFile).trigger('change'); selectOverwriteRulesImport(); selectOverwriteExceptionsRulesImport(); + selectOverwriteConnectorsRulesImport(); cy.get(RULE_IMPORT_MODAL_BUTTON).last().click({ force: true }); cy.get(INPUT_FILE).should('not.exist'); }; diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/import_data_modal/__snapshots__/index.test.tsx.snap index 47dd897f71ffb..3378225f6b602 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/__snapshots__/index.test.tsx.snap @@ -90,13 +90,521 @@ Object {
+
+
+ +
+ +
+
+
+
+ + +
+
+ + + , + "container":
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ImportDataModal should import file, with warnings 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+
+
+
+ +
+

+ title +

+
+
+
+
+

+ description +

+
+
+
+
+ +
+
+
+
+
+
+

+

+
+
+
+
+ 1 connector has sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
+
+ +
+ +
+
+
+
+ + +
+
+
+
+ , + "container":
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ImportDataModal should import file, with warnings but no action_connectors_success_count 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+
+
+
+ +
+

+ title +

+
+
+
+
+

+ description +

+
+
+
+
+ +
+
+
+
+
+
diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/__snapshots__/action_connectors_warning.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/__snapshots__/action_connectors_warning.test.tsx.snap new file mode 100644 index 0000000000000..fba4aa3d0fb96 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/__snapshots__/action_connectors_warning.test.tsx.snap @@ -0,0 +1,678 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ActionConnectorWarnings should not render if importedActionConnectorsCount is falsy and empty warnings array 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+ , + "container":
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ActionConnectorWarnings should not render if importedActionConnectorsCount is truthy and empty warnings array 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+ , + "container":
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ActionConnectorWarnings should render if 1 connectors were imported and use the warning message with the correct imported number 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+
+

+

+
+
+
+
+ 1 connector has sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
+ , + "container":
+
+

+

+
+
+
+
+ 1 connector has sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ActionConnectorWarnings should render if 2 connectors were imported and use the button label when is set 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+
+

+

+
+
+
+
+ 2 connectors have sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
+ , + "container":
+
+

+

+
+
+
+
+ 2 connectors have sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; + +exports[`ActionConnectorWarnings should render if 2 connectors were imported and use the warning message with the correct imported number 1`] = ` +Object { + "asFragment": [Function], + "baseElement": +
+
+

+

+
+
+
+
+ 2 connectors have sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
+ , + "container":
+
+

+

+
+
+
+
+ 2 connectors have sensitive information that requires updates. +
+
+
+
+ +
+
+
+
+
+
, + "debug": [Function], + "findAllByAltText": [Function], + "findAllByDisplayValue": [Function], + "findAllByLabelText": [Function], + "findAllByPlaceholderText": [Function], + "findAllByRole": [Function], + "findAllByTestId": [Function], + "findAllByText": [Function], + "findAllByTitle": [Function], + "findByAltText": [Function], + "findByDisplayValue": [Function], + "findByLabelText": [Function], + "findByPlaceholderText": [Function], + "findByRole": [Function], + "findByTestId": [Function], + "findByText": [Function], + "findByTitle": [Function], + "getAllByAltText": [Function], + "getAllByDisplayValue": [Function], + "getAllByLabelText": [Function], + "getAllByPlaceholderText": [Function], + "getAllByRole": [Function], + "getAllByTestId": [Function], + "getAllByText": [Function], + "getAllByTitle": [Function], + "getByAltText": [Function], + "getByDisplayValue": [Function], + "getByLabelText": [Function], + "getByPlaceholderText": [Function], + "getByRole": [Function], + "getByTestId": [Function], + "getByText": [Function], + "getByTitle": [Function], + "queryAllByAltText": [Function], + "queryAllByDisplayValue": [Function], + "queryAllByLabelText": [Function], + "queryAllByPlaceholderText": [Function], + "queryAllByRole": [Function], + "queryAllByTestId": [Function], + "queryAllByText": [Function], + "queryAllByTitle": [Function], + "queryByAltText": [Function], + "queryByDisplayValue": [Function], + "queryByLabelText": [Function], + "queryByPlaceholderText": [Function], + "queryByRole": [Function], + "queryByTestId": [Function], + "queryByText": [Function], + "queryByTitle": [Function], + "rerender": [Function], + "unmount": [Function], +} +`; diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/action_connectors_warning.test.tsx b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/action_connectors_warning.test.tsx new file mode 100644 index 0000000000000..a71ce2beeaebe --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/action_connectors_warning.test.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { render } from '@testing-library/react'; +import { ActionConnectorWarnings } from '.'; + +jest.mock('../../../lib/kibana/kibana_react', () => ({ + useKibana: jest.fn().mockReturnValue({ + services: { http: { basePath: { prepend: jest.fn() } } }, + }), +})); +describe('ActionConnectorWarnings', () => { + test('should not render if importedActionConnectorsCount is falsy and empty warnings array', () => { + const wrapper = render( + + ); + expect(wrapper).toMatchSnapshot(); + expect(wrapper.queryByTestId('actionConnectorsWarningsCallOut')).not.toBeInTheDocument(); + }); + test('should not render if importedActionConnectorsCount is truthy and empty warnings array', () => { + const wrapper = render( + + ); + expect(wrapper).toMatchSnapshot(); + expect(wrapper.queryByTestId('actionConnectorsWarningsCallOut')).not.toBeInTheDocument(); + }); + test('should render if 1 connectors were imported and use the warning message with the correct imported number', () => { + const wrapper = render( + + ); + const { getByTestId } = wrapper; + expect(wrapper).toMatchSnapshot(); + expect(getByTestId('actionConnectorsWarningsCallOutTitle').textContent).toBe( + '1 connector imported' + ); + expect(getByTestId('actionConnectorsWarningsCallOutMessage').textContent).toBe( + '1 connector has sensitive information that requires updates.' + ); + }); + test('should render if 2 connectors were imported and use the warning message with the correct imported number', () => { + const wrapper = render( + + ); + const { getByTestId } = wrapper; + expect(wrapper).toMatchSnapshot(); + expect(getByTestId('actionConnectorsWarningsCallOutTitle').textContent).toBe( + '2 connectors imported' + ); + expect(getByTestId('actionConnectorsWarningsCallOutMessage').textContent).toBe( + '2 connectors have sensitive information that requires updates.' + ); + expect(getByTestId('actionConnectorsWarningsCallOutButton').textContent).toBe( + 'Go to connectors' + ); + }); + test('should render if 2 connectors were imported and use the button label when is set', () => { + const wrapper = render( + + ); + const { getByTestId } = wrapper; + expect(wrapper).toMatchSnapshot(); + expect(getByTestId('actionConnectorsWarningsCallOutTitle').textContent).toBe( + '2 connectors imported' + ); + expect(getByTestId('actionConnectorsWarningsCallOutMessage').textContent).toBe( + '2 connectors have sensitive information that requires updates.' + ); + expect(getByTestId('actionConnectorsWarningsCallOutButton').textContent).toBe('Connectors'); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/index.tsx b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/index.tsx new file mode 100644 index 0000000000000..ad72893f66d87 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/action_connectors_warning/index.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import type { FC } from 'react'; + +import { EuiButton, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; +import type { WarningSchema } from '../../../../../common/detection_engine/schemas/response'; +import { useKibana } from '../../../lib/kibana/kibana_react'; +import * as i18n from '../translations'; + +interface ActionConnectorWarningsComponentProps { + actionConnectorsWarnings: WarningSchema[]; + importedActionConnectorsCount?: number; +} +const ActionConnectorWarningsComponent: FC = ({ + actionConnectorsWarnings, + importedActionConnectorsCount, +}) => { + const { http } = useKibana().services; + + if (!importedActionConnectorsCount || !actionConnectorsWarnings.length) return null; + const { actionPath, message, buttonLabel } = actionConnectorsWarnings[0]; + + return ( + + {i18n.ACTION_CONNECTORS_WARNING_TITLE(importedActionConnectorsCount)} + + } + color="warning" + > + + + {message} + + + + + {buttonLabel || i18n.ACTION_CONNECTORS_WARNING_BUTTON} + + + + + + ); +}; + +ActionConnectorWarningsComponent.displayName = 'ActionConnectorWarningsComponent'; + +export const ActionConnectorWarnings = React.memo(ActionConnectorWarningsComponent); + +ActionConnectorWarnings.displayName = 'ActionConnectorWarnings'; diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.test.tsx index 515019785534f..1986539e578e0 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.test.tsx @@ -12,6 +12,11 @@ import { ImportDataModalComponent } from '.'; jest.mock('../../lib/kibana'); +jest.mock('../../lib/kibana/kibana_react', () => ({ + useKibana: jest.fn().mockReturnValue({ + services: { http: { basePath: { prepend: jest.fn() } } }, + }), +})); jest.mock('../../hooks/use_app_toasts', () => ({ useAppToasts: jest.fn().mockReturnValue({ addError: jest.fn(), @@ -25,6 +30,9 @@ const importData = jest.fn().mockReturnValue({ success: true, errors: [] }); const file = new File(['file'], 'image1.png', { type: 'image/png' }); describe('ImportDataModal', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); test('renders correctly against snapshot', () => { const wrapper = render( { successMessage={jest.fn((totalCount) => 'successMessage')} title="title" showExceptionsCheckBox={true} + showActionConnectorsCheckBox={true} /> ); const overwriteCheckbox: HTMLInputElement = queryByTestId( - 'import-data-modal-checkbox-label' + 'importDataModalCheckboxLabel' ) as HTMLInputElement; const exceptionCheckbox: HTMLInputElement = queryByTestId( - 'import-data-modal-exceptions-checkbox-label' + 'importDataModalExceptionsCheckboxLabel' + ) as HTMLInputElement; + const connectorsCheckbox: HTMLInputElement = queryByTestId( + 'importDataModalActionConnectorsCheckbox' ) as HTMLInputElement; await waitFor(() => fireEvent.click(overwriteCheckbox)); await waitFor(() => fireEvent.click(exceptionCheckbox)); + await waitFor(() => fireEvent.click(connectorsCheckbox)); await waitFor(() => fireEvent.change(queryByTestId('rule-file-picker') as HTMLInputElement, { @@ -111,6 +124,7 @@ describe('ImportDataModal', () => { ); expect(overwriteCheckbox.checked).toBeTruthy(); expect(exceptionCheckbox.checked).toBeTruthy(); + expect(connectorsCheckbox.checked).toBeTruthy(); await waitFor(() => { fireEvent.click(queryByTestId('import-data-modal-button') as HTMLButtonElement); @@ -120,6 +134,7 @@ describe('ImportDataModal', () => { expect(overwriteCheckbox.checked).toBeFalsy(); expect(exceptionCheckbox.checked).toBeFalsy(); + expect(connectorsCheckbox.checked).toBeFalsy(); }); test('should uncheck the selected checkboxes after closing the Flyout', async () => { const { queryByTestId, getAllByRole } = render( @@ -138,20 +153,25 @@ describe('ImportDataModal', () => { successMessage={jest.fn((totalCount) => 'successMessage')} title="title" showExceptionsCheckBox={true} + showActionConnectorsCheckBox={true} /> ); const closeButton = getAllByRole('button')[0]; const overwriteCheckbox: HTMLInputElement = queryByTestId( - 'import-data-modal-checkbox-label' + 'importDataModalCheckboxLabel' ) as HTMLInputElement; const exceptionCheckbox: HTMLInputElement = queryByTestId( - 'import-data-modal-exceptions-checkbox-label' + 'importDataModalExceptionsCheckboxLabel' + ) as HTMLInputElement; + const connectorsCheckbox: HTMLInputElement = queryByTestId( + 'importDataModalActionConnectorsCheckbox' ) as HTMLInputElement; await waitFor(() => fireEvent.click(overwriteCheckbox)); await waitFor(() => fireEvent.click(exceptionCheckbox)); + await waitFor(() => fireEvent.click(connectorsCheckbox)); await waitFor(() => fireEvent.change(queryByTestId('rule-file-picker') as HTMLInputElement, { @@ -168,5 +188,91 @@ describe('ImportDataModal', () => { expect(overwriteCheckbox.checked).toBeFalsy(); expect(exceptionCheckbox.checked).toBeFalsy(); + expect(connectorsCheckbox.checked).toBeFalsy(); + }); + + test('should import file, with warnings but no action_connectors_success_count', async () => { + const importWithWarning = jest.fn().mockReturnValue({ + ...importData(), + action_connectors_warnings: [ + { message: 'message', actionPath: 'path', buttonLabel: 'buttonLabel' }, + ], + action_connectors_success_count: 0, + }); + const wrapper = render( + 'successMessage')} + title="title" + /> + ); + const { queryByTestId } = wrapper; + await waitFor(() => { + fireEvent.change(queryByTestId('rule-file-picker') as HTMLInputElement, { + target: { files: [file] }, + }); + }); + await waitFor(() => { + fireEvent.click(queryByTestId('import-data-modal-button') as HTMLButtonElement); + }); + expect(wrapper).toMatchSnapshot(); + expect(queryByTestId('actionConnectorsWarningsCallOut')).not.toBeInTheDocument(); + expect(importWithWarning).toHaveBeenCalled(); + expect(closeModal).not.toHaveBeenCalled(); + expect(importComplete).toHaveBeenCalled(); + }); + test('should import file, with warnings', async () => { + const importWithWarning = jest.fn().mockReturnValue({ + ...importData(), + action_connectors_warnings: [ + { + message: '1 connector has sensitive information that requires updates.', + actionPath: 'path', + buttonLabel: 'buttonLabel', + }, + ], + action_connectors_success_count: 1, + }); + const wrapper = render( + 'successMessage')} + title="title" + /> + ); + const { queryByTestId } = wrapper; + await waitFor(() => { + fireEvent.change(queryByTestId('rule-file-picker') as HTMLInputElement, { + target: { files: [file] }, + }); + }); + await waitFor(() => { + fireEvent.click(queryByTestId('import-data-modal-button') as HTMLButtonElement); + }); + expect(wrapper).toMatchSnapshot(); + expect(queryByTestId('actionConnectorsWarningsCallOut')).toBeInTheDocument(); + expect(importWithWarning).toHaveBeenCalled(); + expect(importComplete).toHaveBeenCalled(); + expect(closeModal).not.toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.tsx index a291265bc234d..1b2cf3e516e5e 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/index.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import React, { useCallback, useState } from 'react'; import { EuiButton, EuiButtonEmpty, @@ -18,7 +19,7 @@ import { EuiSpacer, EuiText, } from '@elastic/eui'; -import React, { useCallback, useState } from 'react'; +import type { WarningSchema } from '../../../../common/detection_engine/schemas/response'; import type { ImportDataResponse, @@ -27,6 +28,7 @@ import type { import { useAppToasts } from '../../hooks/use_app_toasts'; import * as i18n from './translations'; import { showToasterMessage } from './utils'; +import { ActionConnectorWarnings } from './action_connectors_warning'; interface ImportDataModalProps { checkBoxLabel: string; @@ -43,6 +45,7 @@ interface ImportDataModalProps { subtitle: string; successMessage: (totalCount: number) => string; title: string; + showActionConnectorsCheckBox?: boolean; } /** @@ -59,6 +62,7 @@ export const ImportDataModalComponent = ({ importData, showCheckBox = true, showExceptionsCheckBox = false, + showActionConnectorsCheckBox = false, showModal, submitBtnText, subtitle, @@ -69,15 +73,35 @@ export const ImportDataModalComponent = ({ const [isImporting, setIsImporting] = useState(false); const [overwrite, setOverwrite] = useState(false); const [overwriteExceptions, setOverwriteExceptions] = useState(false); + const [overwriteActionConnectors, setOverwriteActionConnectors] = useState(false); const { addError, addSuccess } = useAppToasts(); - + const [actionConnectorsWarnings, setActionConnectorsWarnings] = useState( + [] + ); + const [importedActionConnectorsCount, setImportedActionConnectorsCount] = useState< + number | undefined + >(0); const cleanupAndCloseModal = useCallback(() => { - setIsImporting(false); - setSelectedFiles(null); closeModal(); setOverwrite(false); setOverwriteExceptions(false); - }, [setIsImporting, setSelectedFiles, closeModal, setOverwrite, setOverwriteExceptions]); + setOverwriteActionConnectors(false); + setActionConnectorsWarnings([]); + }, [closeModal, setOverwrite, setOverwriteExceptions]); + + const onImportComplete = useCallback( + (callCleanup: boolean) => { + setIsImporting(false); + setSelectedFiles(null); + importComplete(); + + if (callCleanup) { + importComplete(); + cleanupAndCloseModal(); + } + }, + [cleanupAndCloseModal, importComplete] + ); const importDataCallback = useCallback(async () => { if (selectedFiles != null) { @@ -85,25 +109,28 @@ export const ImportDataModalComponent = ({ const abortCtrl = new AbortController(); try { - const importResponse = await importData({ + const { action_connectors_warnings: warnings, ...importResponse } = await importData({ fileToImport: selectedFiles[0], overwrite, overwriteExceptions, + overwriteActionConnectors, signal: abortCtrl.signal, }); + const connectorsCount = importResponse.action_connectors_success_count; + setActionConnectorsWarnings(warnings as WarningSchema[]); + setImportedActionConnectorsCount(connectorsCount); showToasterMessage({ importResponse, exceptionsIncluded: showExceptionsCheckBox, + actionConnectorsIncluded: showActionConnectorsCheckBox, successMessage, errorMessage, errorMessageDetailed: failedDetailed, addError, addSuccess, }); - - importComplete(); - cleanupAndCloseModal(); + onImportComplete(!warnings?.length); } catch (error) { cleanupAndCloseModal(); addError(error, { title: errorMessage(1) }); @@ -114,13 +141,15 @@ export const ImportDataModalComponent = ({ importData, overwrite, overwriteExceptions, + overwriteActionConnectors, showExceptionsCheckBox, successMessage, errorMessage, failedDetailed, addError, addSuccess, - importComplete, + showActionConnectorsCheckBox, + onImportComplete, cleanupAndCloseModal, ]); @@ -136,6 +165,9 @@ export const ImportDataModalComponent = ({ setOverwriteExceptions((shouldOverwrite) => !shouldOverwrite); }, []); + const handleActionConnectorsCheckboxClick = useCallback(() => { + setOverwriteActionConnectors((shouldOverwrite) => !shouldOverwrite); + }, []); return ( <> {showModal && ( @@ -163,24 +195,41 @@ export const ImportDataModalComponent = ({ isLoading={isImporting} /> + + + + + {showCheckBox && ( <> {showExceptionsCheckBox && ( )} + {showActionConnectorsCheckBox && ( + + )} )} diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/translations.ts b/x-pack/plugins/security_solution/public/common/components/import_data_modal/translations.ts index 763396ae62d79..11ae7dd972a57 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/translations.ts +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/translations.ts @@ -20,7 +20,12 @@ export const OVERWRITE_EXCEPTIONS_LABEL = i18n.translate( defaultMessage: 'Overwrite existing exception lists with conflicting "list_id"', } ); - +export const OVERWRITE_ACTION_CONNECTORS_LABEL = i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.overwriteActionConnectorsLabel', + { + defaultMessage: 'Overwrite existing connectors with conflicting action "id"', + } +); export const SUCCESSFULLY_IMPORTED_EXCEPTIONS = (totalExceptions: number) => i18n.translate( 'xpack.securitySolution.detectionEngine.components.importRuleModal.exceptionsSuccessLabel', @@ -30,6 +35,15 @@ export const SUCCESSFULLY_IMPORTED_EXCEPTIONS = (totalExceptions: number) => 'Successfully imported {totalExceptions} {totalExceptions, plural, =1 {exception} other {exceptions}}.', } ); +export const SUCCESSFULLY_IMPORTED_CONNECTORS = (totalConnectors: number) => + i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.connectorsSuccessLabel', + { + values: { totalConnectors }, + defaultMessage: + 'Successfully imported {totalConnectors} {totalConnectors, plural, =1 {connector} other {connectors}}.', + } + ); export const IMPORT_FAILED = (totalExceptions: number) => i18n.translate( @@ -40,3 +54,36 @@ export const IMPORT_FAILED = (totalExceptions: number) => 'Failed to import {totalExceptions} {totalExceptions, plural, =1 {exception} other {exceptions}}', } ); +export const IMPORT_CONNECTORS_FAILED = (totalConnectors: number) => + i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.importConnectorsFailedLabel', + { + values: { totalConnectors }, + defaultMessage: + 'Failed to import {totalConnectors} {totalConnectors, plural, =1 {connector} other {connectors}}', + } + ); + +export const ACTION_CONNECTORS_WARNING_TITLE = (totalConnectors: number) => + i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.actionConnectorsWarningTitle', + { + values: { totalConnectors }, + defaultMessage: + '{totalConnectors} {totalConnectors, plural, =1 {connector} other {connectors}} imported', + } + ); + +export const ACTION_CONNECTORS_WARNING_BUTTON = i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.actionConnectorsWarningButton', + { + defaultMessage: 'Go to connectors', + } +); + +export const ACTION_CONNECTORS_ADDITIONAL_PRIVILEGES = i18n.translate( + 'xpack.securitySolution.detectionEngine.components.importRuleModal.actionConnectorsAdditionalPrivilegesError', + { + defaultMessage: 'You need additional privileges to import rules with actions.', + } +); diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.test.ts b/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.test.ts index b5609a79eba6c..6b432edfb5bd1 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.test.ts +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.test.ts @@ -9,7 +9,7 @@ import { showToasterMessage } from './utils'; describe('showToasterMessage', () => { describe('exceptionsIncluded is false', () => { - it('displays main success message if import was successful', () => { + it('should display main success message if import was successful', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -20,6 +20,7 @@ describe('showToasterMessage', () => { errors: [], }, exceptionsIncluded: false, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -32,7 +33,7 @@ describe('showToasterMessage', () => { expect(addError).not.toHaveBeenCalled(); }); - it('displays main error message if import was not successful', () => { + it('should display main error message if import was not successful', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -51,6 +52,7 @@ describe('showToasterMessage', () => { ], }, exceptionsIncluded: false, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -67,7 +69,7 @@ describe('showToasterMessage', () => { }); describe('exceptionsIncluded is true', () => { - it('displays success message for rules and exceptions if both succeed', () => { + it('should display success message for rules and exceptions if both succeed', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -82,6 +84,7 @@ describe('showToasterMessage', () => { exceptions_success_count: 1, }, exceptionsIncluded: true, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -95,7 +98,7 @@ describe('showToasterMessage', () => { expect(addError).not.toHaveBeenCalled(); }); - it('displays error message for rules and exceptions if both fail', () => { + it('should display error message for rules and exceptions if both fail', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -133,6 +136,7 @@ describe('showToasterMessage', () => { exceptions_success_count: 0, }, exceptionsIncluded: true, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -154,7 +158,7 @@ describe('showToasterMessage', () => { expect(addSuccess).not.toHaveBeenCalled(); }); - it('displays only a rule toaster if no exceptions were imported', () => { + it('should display only a rule toaster if no exceptions were imported', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -169,6 +173,7 @@ describe('showToasterMessage', () => { exceptions_success_count: 0, }, exceptionsIncluded: true, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -181,7 +186,7 @@ describe('showToasterMessage', () => { expect(addError).not.toHaveBeenCalled(); }); - it('displays only an exceptions toaster if no rules were imported', () => { + it('should display only an exceptions toaster if no rules were imported', () => { const addError = jest.fn(); const addSuccess = jest.fn(); @@ -196,6 +201,7 @@ describe('showToasterMessage', () => { exceptions_success_count: 1, }, exceptionsIncluded: true, + actionConnectorsIncluded: false, successMessage: (msg) => `success: ${msg}`, errorMessage: (msg) => `error: ${msg}`, errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, @@ -208,4 +214,277 @@ describe('showToasterMessage', () => { expect(addError).not.toHaveBeenCalled(); }); }); + + describe('actionConnectorsIncluded is false', () => { + it('should display main success message if import was successful', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: true, + success_count: 1, + errors: [], + }, + exceptionsIncluded: false, + actionConnectorsIncluded: false, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addSuccess).toHaveBeenCalledTimes(1); + expect(addSuccess).toHaveBeenCalledWith('success: 1'); + expect(addError).not.toHaveBeenCalled(); + }); + + it('should display main error message if import was not successful', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: false, + success_count: 0, + errors: [ + { + rule_id: 'rule_id', + error: { + status_code: 400, + message: 'an error message', + }, + }, + ], + }, + exceptionsIncluded: false, + actionConnectorsIncluded: false, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addError).toHaveBeenCalledTimes(1); + expect(addError).toHaveBeenCalledWith(new Error('errorDetailed: an error message'), { + title: 'error: 1', + }); + expect(addSuccess).not.toHaveBeenCalled(); + }); + }); + describe('actionConnectorsIncluded is true', () => { + it('should display success message for rules and connectors if both succeed', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: true, + success_count: 1, + rules_count: 1, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_errors: [], + action_connectors_success_count: 1, + }, + exceptionsIncluded: false, + actionConnectorsIncluded: true, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addSuccess).toHaveBeenCalledTimes(2); + expect(addSuccess).toHaveBeenNthCalledWith(1, 'success: 1'); + expect(addSuccess).toHaveBeenNthCalledWith(2, 'Successfully imported 1 connector.'); + expect(addError).not.toHaveBeenCalled(); + }); + + it('should display 1 error message for rules and connectors even when both fail', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: false, + success_count: 1, + rules_count: 2, + action_connectors_success: false, + errors: [ + { + rule_id: 'rule_id', + error: { + status_code: 400, + message: 'an error message', + }, + }, + ], + action_connectors_errors: [ + { + rule_id: 'rule_id', + error: { + status_code: 400, + message: 'an error message', + }, + }, + { + rule_id: 'rule_id_1', + error: { + status_code: 400, + message: 'another error message', + }, + }, + ], + exceptions_success: true, + exceptions_success_count: 0, + }, + exceptionsIncluded: false, + actionConnectorsIncluded: true, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addError).toHaveBeenCalledTimes(1); + + expect(addError).toHaveBeenCalledWith( + new Error('errorDetailed: an error message. errorDetailed: another error message'), + { + title: 'Failed to import 2 connectors', + } + ); + expect(addSuccess).not.toHaveBeenCalled(); + }); + + it('should display only a rule toaster if no connectors were imported', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: true, + success_count: 1, + rules_count: 1, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_errors: [], + action_connectors_success: true, + }, + exceptionsIncluded: true, + actionConnectorsIncluded: false, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addSuccess).toHaveBeenCalledTimes(1); + expect(addSuccess).toHaveBeenNthCalledWith(1, 'success: 1'); + expect(addError).not.toHaveBeenCalled(); + }); + + it('should display only a connector toaster if no rules were imported', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: true, + success_count: 0, + rules_count: 0, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success_count: 1, + action_connectors_success: true, + }, + exceptionsIncluded: true, + actionConnectorsIncluded: true, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addSuccess).toHaveBeenCalledTimes(1); + expect(addSuccess).toHaveBeenNthCalledWith(1, 'Successfully imported 1 connector.'); + expect(addError).not.toHaveBeenCalled(); + }); + it('should display the user friendly message in case of additional privileges', () => { + const addError = jest.fn(); + const addSuccess = jest.fn(); + + showToasterMessage({ + importResponse: { + success: false, + success_count: 1, + rules_count: 2, + action_connectors_success: false, + errors: [ + { + rule_id: 'unknown', + error: { + status_code: 403, + message: + 'You may not have actions privileges required to import rules with actions', + }, + }, + ], + action_connectors_errors: [ + { + rule_id: 'unknown', + error: { + status_code: 403, + message: + 'You may not have actions privileges required to import rules with actions', + }, + }, + { + rule_id: 'unknown', + error: { + status_code: 403, + message: + 'You may not have actions privileges required to import rules with actions', + }, + }, + ], + exceptions_success: true, + exceptions_success_count: 0, + }, + exceptionsIncluded: false, + actionConnectorsIncluded: true, + successMessage: (msg) => `success: ${msg}`, + errorMessage: (msg) => `error: ${msg}`, + errorMessageDetailed: (msg) => `errorDetailed: ${msg}`, + addError, + addSuccess, + }); + + expect(addError).toHaveBeenCalledTimes(1); + + expect(addError).toHaveBeenCalledWith( + new Error( + 'errorDetailed: You need additional privileges to import rules with actions.. errorDetailed: You need additional privileges to import rules with actions.' + ), + { + title: 'Failed to import 2 connectors', + } + ); + expect(addSuccess).not.toHaveBeenCalled(); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.ts b/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.ts index 8211c01a7bffa..92c6f9f666e4b 100644 --- a/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.ts +++ b/x-pack/plugins/security_solution/public/common/components/import_data_modal/utils.ts @@ -30,9 +30,32 @@ export const formatError = ( return error; }; +const mapErrorMessageToUserMessage = ( + actionConnectorsErrors: Array +) => { + return actionConnectorsErrors.map((connectorError) => { + const { error } = connectorError; + const { status_code: statusCode, message: originalMessage } = error || {}; + let message; + switch (statusCode) { + case 403: + message = i18n.ACTION_CONNECTORS_ADDITIONAL_PRIVILEGES; + break; + + default: + message = originalMessage; + + break; + } + + return { ...connectorError, error: { ...error, message } }; + }); +}; + export const showToasterMessage = ({ importResponse, exceptionsIncluded, + actionConnectorsIncluded, successMessage, errorMessage, errorMessageDetailed, @@ -41,53 +64,68 @@ export const showToasterMessage = ({ }: { importResponse: ImportDataResponse; exceptionsIncluded: boolean; + actionConnectorsIncluded: boolean; successMessage: (totalCount: number) => string; errorMessage: (totalCount: number) => string; errorMessageDetailed: (message: string) => string; addError: (error: unknown, options: ErrorToastOptions) => Toast; addSuccess: (toastOrTitle: ToastInput, options?: ToastOptions | undefined) => Toast; }) => { - // if import includes exceptions - if (exceptionsIncluded) { - // rules response actions - if (importResponse.success && importResponse.success_count > 0) { - addSuccess(successMessage(importResponse.success_count)); - } - - if (importResponse.errors.length > 0 && importResponse.rules_count) { - const error = formatError(errorMessageDetailed, importResponse, importResponse.errors); - addError(error, { - title: errorMessage(importResponse.rules_count - importResponse.success_count), - }); - } - - // exceptions response actions + if (importResponse.success) { + if (importResponse.success_count > 0) addSuccess(successMessage(importResponse.success_count)); if ( + exceptionsIncluded && importResponse.exceptions_success && importResponse.exceptions_success_count != null && importResponse.exceptions_success_count > 0 ) { addSuccess(i18n.SUCCESSFULLY_IMPORTED_EXCEPTIONS(importResponse.exceptions_success_count)); } + if ( + actionConnectorsIncluded && + importResponse.action_connectors_success && + importResponse.action_connectors_success_count != null && + importResponse.action_connectors_success_count > 0 + ) { + addSuccess( + i18n.SUCCESSFULLY_IMPORTED_CONNECTORS(importResponse.action_connectors_success_count) + ); + } + return; + } + + if (importResponse.errors.length > 0) { + if ( + actionConnectorsIncluded && + importResponse.action_connectors_errors != null && + importResponse.action_connectors_errors.length > 0 + ) { + const userErrorMessages = mapErrorMessageToUserMessage( + importResponse.action_connectors_errors + ); + const connectorError = formatError(errorMessageDetailed, importResponse, userErrorMessages); - if (importResponse.exceptions_errors != null && importResponse.exceptions_errors.length > 0) { - const error = formatError( + return addError(connectorError, { + title: i18n.IMPORT_CONNECTORS_FAILED(userErrorMessages.length), + }); + } + const ruleError = formatError(errorMessageDetailed, importResponse, importResponse.errors); + addError(ruleError, { title: errorMessage(importResponse.errors.length) }); + + if ( + exceptionsIncluded && + importResponse.exceptions_errors != null && + importResponse.exceptions_errors.length > 0 + ) { + const exceptionError = formatError( errorMessageDetailed, importResponse, importResponse.exceptions_errors ); - addError(error, { title: i18n.IMPORT_FAILED(importResponse.exceptions_errors.length) }); - } - } else { - // rules response actions - if (importResponse.success) { - addSuccess(successMessage(importResponse.success_count)); - } - - if (importResponse.errors.length > 0) { - const error = formatError(errorMessageDetailed, importResponse, importResponse.errors); - addError(error, { title: errorMessage(importResponse.errors.length) }); + addError(exceptionError, { + title: i18n.IMPORT_FAILED(importResponse.exceptions_errors.length), + }); } } }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.test.ts index bf1ac32d2bc78..a1d59efa4c30a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.test.ts @@ -493,6 +493,7 @@ describe('Detections Rules API', () => { }, query: { overwrite: false, + overwrite_action_connectors: false, overwrite_exceptions: false, }, }); @@ -510,6 +511,7 @@ describe('Detections Rules API', () => { query: { overwrite: true, overwrite_exceptions: false, + overwrite_action_connectors: false, }, }); }); @@ -523,6 +525,10 @@ describe('Detections Rules API', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); const resp = await importRules({ fileToImport, signal: abortCtrl.signal }); expect(resp).toEqual({ @@ -533,6 +539,10 @@ describe('Detections Rules API', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.ts index c867b6b3fd792..aec165167db59 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/api/api.ts @@ -341,6 +341,7 @@ export const importRules = async ({ fileToImport, overwrite = false, overwriteExceptions = false, + overwriteActionConnectors = false, signal, }: ImportDataProps): Promise => { const formData = new FormData(); @@ -351,7 +352,11 @@ export const importRules = async ({ { method: 'POST', headers: { 'Content-Type': undefined }, - query: { overwrite, overwrite_exceptions: overwriteExceptions }, + query: { + overwrite, + overwrite_exceptions: overwriteExceptions, + overwrite_action_connectors: overwriteActionConnectors, + }, body: formData, signal, } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts index 6b520659090d9..246a000356761 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts @@ -29,6 +29,7 @@ import { import type { NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; import { PositiveInteger } from '@kbn/securitysolution-io-ts-types'; +import type { WarningSchema } from '../../../../common/detection_engine/schemas/response'; import { RuleExecutionSummary } from '../../../../common/detection_engine/rule_monitoring'; import { AlertSuppression, @@ -267,6 +268,7 @@ export interface ImportDataProps { fileToImport: File; overwrite?: boolean; overwriteExceptions?: boolean; + overwriteActionConnectors?: boolean; signal: AbortSignal; } @@ -304,6 +306,10 @@ export interface ImportDataResponse { exceptions_success?: boolean; exceptions_success_count?: number; exceptions_errors?: ExceptionsImportError[]; + action_connectors_success?: boolean; + action_connectors_success_count?: number; + action_connectors_errors?: Array; + action_connectors_warnings?: WarningSchema[]; } export interface ExportDocumentsProps { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/pages/rule_management/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/pages/rule_management/index.tsx index f02075809a46d..a26517e447d7a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/pages/rule_management/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/pages/rule_management/index.tsx @@ -108,6 +108,7 @@ const RulesPageComponent: React.FC = () => { title={i18n.IMPORT_RULE} showExceptionsCheckBox showCheckBox + showActionConnectorsCheckBox /> diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts index 2234f09cb1888..e3fabc5366980 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts @@ -326,12 +326,18 @@ export const performBulkActionRoute = ( 'alerting', 'licensing', 'lists', + 'actions', ]); const rulesClient = ctx.alerting.getRulesClient(); const exceptionsClient = ctx.lists?.getExceptionListClient(); const savedObjectsClient = ctx.core.savedObjects.client; + const { getExporter, getClient } = (await ctx.core).savedObjects; + const client = getClient({ includedHiddenTypes: ['action'] }); + + const exporter = getExporter(client); + const mlAuthz = buildMlAuthz({ license: ctx.licensing.license, ml, @@ -556,10 +562,12 @@ export const performBulkActionRoute = ( exceptionsClient, savedObjectsClient, rules.map(({ params }) => ({ rule_id: params.ruleId })), - logger + logger, + exporter, + request ); - const responseBody = `${exported.rulesNdjson}${exported.exceptionLists}${exported.exportDetails}`; + const responseBody = `${exported.rulesNdjson}${exported.exceptionLists}${exported.actionConnectors}${exported.exportDetails}`; return response.ok({ headers: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts index ec96f38f5f86e..a1a5f798fc002 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts @@ -45,8 +45,14 @@ export const exportRulesRoute = ( const siemResponse = buildSiemResponse(response); const rulesClient = (await context.alerting).getRulesClient(); const exceptionsClient = (await context.lists)?.getExceptionListClient(); - const savedObjectsClient = (await context.core).savedObjects.client; + const { + getExporter, + getClient, + client: savedObjectsClient, + } = (await context.core).savedObjects; + const client = getClient({ includedHiddenTypes: ['action'] }); + const actionsExporter = getExporter(client); try { const exportSizeLimit = config.maxRuleImportExportSize; if (request.body?.objects != null && request.body.objects.length > exportSizeLimit) { @@ -66,20 +72,29 @@ export const exportRulesRoute = ( } } - const exportedRulesAndExceptions = + const exportedRulesAndReferences = request.body?.objects != null ? await getExportByObjectIds( rulesClient, exceptionsClient, savedObjectsClient, request.body.objects, - logger + logger, + actionsExporter, + request ) - : await getExportAll(rulesClient, exceptionsClient, savedObjectsClient, logger); + : await getExportAll( + rulesClient, + exceptionsClient, + savedObjectsClient, + logger, + actionsExporter, + request + ); const responseBody = request.query.exclude_export_details - ? exportedRulesAndExceptions.rulesNdjson - : `${exportedRulesAndExceptions.rulesNdjson}${exportedRulesAndExceptions.exceptionLists}${exportedRulesAndExceptions.exportDetails}`; + ? exportedRulesAndReferences.rulesNdjson + : `${exportedRulesAndReferences.rulesNdjson}${exportedRulesAndReferences.exceptionLists}${exportedRulesAndReferences.actionConnectors}${exportedRulesAndReferences.exportDetails}`; return response.ok({ headers: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts index 3d466d2505074..bb1bb7d55244a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.test.ts @@ -107,6 +107,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -146,6 +150,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -171,6 +179,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -196,6 +208,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -218,6 +234,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); }); @@ -242,6 +262,10 @@ describe('Import rules route', () => { rules_count: 2, exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -262,6 +286,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -302,6 +330,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -332,6 +364,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -353,6 +389,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); }); @@ -387,6 +427,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); @@ -407,6 +451,10 @@ describe('Import rules route', () => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [], }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts index e306caac12194..ddcbf2cbdd6e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts @@ -16,7 +16,6 @@ import type { ImportQuerySchemaDecoded } from '@kbn/securitysolution-io-ts-types import { importQuerySchema } from '@kbn/securitysolution-io-ts-types'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../../../common/constants'; -import type { RuleToImport } from '../../../../../../../common/detection_engine/rule_management'; import { ImportRulesResponse } from '../../../../../../../common/detection_engine/rule_management'; import type { SecuritySolutionPluginRouter } from '../../../../../../types'; @@ -28,7 +27,6 @@ import { isBulkError, isImportRegular, buildSiemResponse } from '../../../../rou import { getTupleDuplicateErrorsAndUniqueRules, - getInvalidConnectors, migrateLegacyActionsIds, } from '../../../utils/utils'; import { createRulesAndExceptionsStreamFromNdJson } from '../../../logic/import/create_rules_stream_from_ndjson'; @@ -38,6 +36,7 @@ import { importRules as importRulesHelper } from '../../../logic/import/import_r import { getReferencedExceptionLists } from '../../../logic/import/gather_referenced_exceptions'; import { importRuleExceptions } from '../../../logic/import/import_rule_exceptions'; import type { HapiReadableStream } from '../../../logic/import/hapi_readable_stream'; +import { importRuleActionConnectors } from '../../../logic/import/action_connectors/import_rule_action_connectors'; const CHUNK_PARSED_OBJECT_SIZE = 50; @@ -81,6 +80,8 @@ export const importRulesRoute = ( const actionSOClient = ctx.core.savedObjects.getClient({ includedHiddenTypes: ['action'], }); + const actionsImporter = ctx.core.savedObjects.getImporter(actionSOClient); + const savedObjectsClient = ctx.core.savedObjects.client; const exceptionsClient = ctx.lists?.getExceptionListClient(); @@ -104,7 +105,7 @@ export const importRulesRoute = ( // parse file to separate out exceptions from rules const readAllStream = createRulesAndExceptionsStreamFromNdJson(objectLimit); - const [{ exceptions, rules }] = await createPromiseFromStreams< + const [{ exceptions, rules, actionConnectors }] = await createPromiseFromStreams< RuleExceptionsPromiseFromStreams[] >([request.body.file as HapiReadableStream, ...readAllStream]); @@ -119,7 +120,6 @@ export const importRulesRoute = ( overwrite: request.query.overwrite_exceptions, maxExceptionsImportSize: objectLimit, }); - // report on duplicate rules const [duplicateIdErrors, parsedObjectsWithoutDuplicateErrors] = getTupleDuplicateErrorsAndUniqueRules(rules, request.query.overwrite); @@ -129,20 +129,23 @@ export const importRulesRoute = ( actionSOClient ); - let parsedRules; - let actionErrors: BulkError[] = []; - const actualRules = rules.filter((rule): rule is RuleToImport => !(rule instanceof Error)); - - if (actualRules.some((rule) => rule.actions && rule.actions.length > 0)) { - const [nonExistentActionErrors, uniqueParsedObjects] = await getInvalidConnectors( - migratedParsedObjectsWithoutDuplicateErrors, - actionsClient - ); - parsedRules = uniqueParsedObjects; - actionErrors = nonExistentActionErrors; - } else { - parsedRules = migratedParsedObjectsWithoutDuplicateErrors; - } + // import actions-connectors + const { + successCount: actionConnectorSuccessCount, + success: actionConnectorSuccess, + warnings: actionConnectorWarnings, + errors: actionConnectorErrors, + } = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter, + rules: migratedParsedObjectsWithoutDuplicateErrors, + overwrite: request.query.overwrite_action_connectors, + }); + const parsedRules = actionConnectorErrors.length + ? [] + : migratedParsedObjectsWithoutDuplicateErrors; + // gather all exception lists that the imported rules reference const foundReferencedExceptionLists = await getReferencedExceptionLists({ rules: parsedRules, @@ -153,7 +156,7 @@ export const importRulesRoute = ( const importRuleResponse: ImportRuleResponse[] = await importRulesHelper({ ruleChunks: chunkParseObjects, - rulesResponseAcc: [...actionErrors, ...duplicateIdErrors], + rulesResponseAcc: [...actionConnectorErrors, ...duplicateIdErrors], mlAuthz, overwriteRules: request.query.overwrite, rulesClient, @@ -161,6 +164,7 @@ export const importRulesRoute = ( exceptionsClient, spaceId: ctx.securitySolution.getSpaceId(), existingLists: foundReferencedExceptionLists, + allowMissingConnectorSecrets: !!actionConnectors.length, }); const errorsResp = importRuleResponse.filter((resp) => isBulkError(resp)) as BulkError[]; @@ -179,6 +183,10 @@ export const importRulesRoute = ( exceptions_errors: exceptionsErrors, exceptions_success: exceptionsSuccess, exceptions_success_count: exceptionsSuccessCount, + action_connectors_success: actionConnectorSuccess, + action_connectors_success_count: actionConnectorSuccessCount, + action_connectors_errors: actionConnectorErrors, + action_connectors_warnings: actionConnectorWarnings, }; const [validated, errors] = validate(importRules, ImportRulesResponse); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/create_rules.ts index f621542fe433f..416edb039919e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/create_rules.ts @@ -19,6 +19,7 @@ export interface CreateRulesOptions id?: string; immutable?: boolean; defaultEnabled?: boolean; + allowMissingConnectorSecrets?: boolean; } export const createRules = async ({ @@ -27,6 +28,7 @@ export const createRules = async ({ id, immutable = false, defaultEnabled = true, + allowMissingConnectorSecrets, }: CreateRulesOptions): Promise> => { const internalRule = convertCreateAPIToInternalSchema(params, immutable, defaultEnabled); const rule = await rulesClient.create({ @@ -34,6 +36,7 @@ export const createRules = async ({ id, }, data: internalRule, + allowMissingConnectorSecrets, }); // Mute the rule if it is first created with the explicit no actions diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/patch_rules.ts index 656366ec9f92b..45d1a21325abd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/crud/patch_rules.ts @@ -16,12 +16,14 @@ export interface PatchRulesOptions { rulesClient: RulesClient; nextParams: PatchRuleRequestBody; existingRule: RuleAlertType | null | undefined; + allowMissingConnectorSecrets?: boolean; } export const patchRules = async ({ rulesClient, existingRule, nextParams, + allowMissingConnectorSecrets, }: PatchRulesOptions): Promise | null> => { if (existingRule == null) { return null; @@ -32,6 +34,7 @@ export const patchRules = async ({ const update = await rulesClient.update({ id: existingRule.id, data: patchedRule, + allowMissingConnectorSecrets, }); if (nextParams.throttle !== undefined) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts index b7395236a2152..d5511ccf7b1cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts @@ -24,13 +24,17 @@ import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { getExceptionListClientMock } from '@kbn/lists-plugin/server/services/exception_lists/exception_list_client.mock'; import type { loggingSystemMock } from '@kbn/core/server/mocks'; import { requestContextMock } from '../../../routes/__mocks__/request_context'; +import { savedObjectsExporterMock } from '@kbn/core-saved-objects-import-export-server-mocks'; +import { mockRouter } from '@kbn/core-http-router-server-mocks'; +import { Readable } from 'stream'; const exceptionsClient = getExceptionListClientMock(); describe('getExportAll', () => { let logger: ReturnType; const { clients } = requestContextMock.createTools(); - + const exporterMock = savedObjectsExporterMock.create(); + const requestMock = mockRouter.createKibanaRequest(); beforeEach(async () => { clients.savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectsResponse()); }); @@ -55,7 +59,9 @@ describe('getExportAll', () => { rulesClient, exceptionsClient, clients.savedObjectsClient, - logger + logger, + exporterMock, + requestMock ); const rulesJson = JSON.parse(exports.rulesNdjson); const detailsJson = JSON.parse(exports.exportDetails); @@ -114,6 +120,11 @@ describe('getExportAll', () => { missing_exception_lists_count: 0, missing_rules: [], missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], }); }); @@ -133,12 +144,186 @@ describe('getExportAll', () => { rulesClient, exceptionsClient, clients.savedObjectsClient, - logger + logger, + exporterMock, + requestMock ); expect(exports).toEqual({ rulesNdjson: '', exportDetails: getSampleDetailsAsNdjson(details), exceptionLists: '', + actionConnectors: '', + }); + }); + test('it will export with rule and action connectors', async () => { + const rulesClient = rulesClientMock.create(); + const result = getFindResultWithSingleHit(); + const alert = { + ...getRuleMock(getQueryRuleParams()), + actions: [ + { + group: 'default', + id: '123', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + actionTypeId: '.slack', + }, + ], + }; + + alert.params = { + ...alert.params, + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + threat: getThreatMock(), + meta: { someMeta: 'someField' }, + timelineId: 'some-timeline-id', + timelineTitle: 'some-timeline-title', + }; + result.data = [alert]; + rulesClient.find.mockResolvedValue(result); + let eventCount = 0; + const readable = new Readable({ + objectMode: true, + read() { + if (eventCount === 0) { + eventCount += 1; + return this.push({ + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + type: 'action', + updated_at: '2023-01-11T11:30:31.683Z', + created_at: '2023-01-11T11:30:31.683Z', + version: 'WzE2MDYsMV0=', + attributes: { + actionTypeId: '.slack', + name: 'slack', + isMissingSecrets: true, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }); + } + if (eventCount === 1) { + eventCount += 1; + return this.push({ + exportedCount: 1, + missingRefCount: 0, + missingReferences: [], + excludedObjectsCount: 0, + excludedObjects: [], + }); + } + return this.push(null); + }, + }); + + const exporterMockWithConnector = { + exportByObjects: () => jest.fn().mockReturnValueOnce(readable), + + exportByTypes: jest.fn(), + }; + const exports = await getExportAll( + rulesClient, + exceptionsClient, + clients.savedObjectsClient, + logger, + exporterMockWithConnector as never, + requestMock + ); + const rulesJson = JSON.parse(exports.rulesNdjson); + const detailsJson = JSON.parse(exports.exportDetails); + const actionConnectorsJSON = JSON.parse(exports.actionConnectors); + expect(rulesJson).toEqual({ + author: ['Elastic'], + actions: [ + { + group: 'default', + id: '123', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + action_type_id: '.slack', + }, + ], + building_block_type: 'default', + created_at: '2019-12-13T16:40:33.400Z', + updated_at: '2019-12-13T16:40:33.400Z', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + rule_id: 'rule-1', + language: 'kuery', + license: 'Elastic License', + output_index: '.siem-signals', + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + name: 'Detect Root/Admin Users', + query: 'user.name: root or user.name: admin', + references: ['http://example.com', 'https://example.com'], + related_integrations: [], + required_fields: [], + setup: '', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + meta: { someMeta: 'someField' }, + severity: 'high', + severity_mapping: [], + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + threat: getThreatMock(), + throttle: 'rule', + note: '# Investigative notes', + version: 1, + exceptions_list: getListArrayMock(), + }); + expect(detailsJson).toEqual({ + exported_exception_list_count: 1, + exported_exception_list_item_count: 1, + exported_count: 4, + exported_rules_count: 1, + missing_exception_list_item_count: 0, + missing_exception_list_items: [], + missing_exception_lists: [], + missing_exception_lists_count: 0, + missing_rules: [], + missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 1, + missing_action_connection_count: 0, + missing_action_connections: [], + }); + expect(actionConnectorsJSON).toEqual({ + attributes: { + actionTypeId: '.slack', + config: {}, + isMissingSecrets: true, + name: 'slack', + secrets: {}, + }, + coreMigrationVersion: '8.7.0', + created_at: '2023-01-11T11:30:31.683Z', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + migrationVersion: { + action: '8.3.0', + }, + references: [], + type: 'action', + updated_at: '2023-01-11T11:30:31.683Z', + version: 'WzE2MDYsMV0=', }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts index 058559de7db59..88f3e9019d493 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.ts @@ -7,13 +7,14 @@ import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import type { Logger } from '@kbn/core/server'; +import type { ISavedObjectsExporter, KibanaRequest, Logger } from '@kbn/core/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import type { RulesClient, RuleExecutorServices } from '@kbn/alerting-plugin/server'; import { getNonPackagedRules } from '../search/get_existing_prepackaged_rules'; import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { transformAlertsToRules } from '../../utils/utils'; import { getRuleExceptionsForExport } from './get_export_rule_exceptions'; +import { getRuleActionConnectorsForExport } from './get_export_rule_action_connectors'; // eslint-disable-next-line no-restricted-imports import { legacyGetBulkRuleActionsSavedObject } from '../../../rule_actions_legacy'; @@ -22,11 +23,14 @@ export const getExportAll = async ( rulesClient: RulesClient, exceptionsClient: ExceptionListClient | undefined, savedObjectsClient: RuleExecutorServices['savedObjectsClient'], - logger: Logger + logger: Logger, + actionsExporter: ISavedObjectsExporter, + request: KibanaRequest ): Promise<{ rulesNdjson: string; exportDetails: string; exceptionLists: string | null; + actionConnectors: string; }> => { const ruleAlertTypes = await getNonPackagedRules({ rulesClient }); const alertIds = ruleAlertTypes.map((rule) => rule.id); @@ -44,7 +48,15 @@ export const getExportAll = async ( const { exportData: exceptionLists, exportDetails: exceptionDetails } = await getRuleExceptionsForExport(exceptions, exceptionsClient); + // Retrieve Action-Connectors + const { actionConnectors, actionConnectorDetails } = await getRuleActionConnectorsForExport( + rules, + actionsExporter, + request + ); + const rulesNdjson = transformDataToNdjson(rules); - const exportDetails = getExportDetailsNdjson(rules, [], exceptionDetails); - return { rulesNdjson, exportDetails, exceptionLists }; + const exportDetails = getExportDetailsNdjson(rules, [], exceptionDetails, actionConnectorDetails); + + return { rulesNdjson, exportDetails, exceptionLists, actionConnectors }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts index 02b83342cd846..570954ba9c8b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { Readable } from 'stream'; import type { RulesErrors } from './get_export_by_object_ids'; import { getExportByObjectIds, getRulesFromObjects } from './get_export_by_object_ids'; import type { FindHit } from '../../../routes/__mocks__/request_responses'; @@ -22,6 +22,8 @@ import { } from '../../../../../../common/detection_engine/rule_management/mocks'; import { getQueryRuleParams } from '../../../rule_schema/mocks'; import { getExceptionListClientMock } from '@kbn/lists-plugin/server/services/exception_lists/exception_list_client.mock'; +import { savedObjectsExporterMock } from '@kbn/core-saved-objects-import-export-server-mocks'; +import { mockRouter } from '@kbn/core-http-router-server-mocks'; const exceptionsClient = getExceptionListClientMock(); import type { loggingSystemMock } from '@kbn/core/server/mocks'; @@ -30,6 +32,8 @@ import { requestContextMock } from '../../../routes/__mocks__/request_context'; describe('get_export_by_object_ids', () => { let logger: ReturnType; const { clients } = requestContextMock.createTools(); + const exporterMock = savedObjectsExporterMock.create(); + const requestMock = mockRouter.createKibanaRequest(); beforeEach(() => { jest.resetAllMocks(); @@ -50,7 +54,9 @@ describe('get_export_by_object_ids', () => { exceptionsClient, clients.savedObjectsClient, objects, - logger + logger, + exporterMock, + requestMock ); const exportsObj = { rulesNdjson: JSON.parse(exports.rulesNdjson), @@ -112,6 +118,11 @@ describe('get_export_by_object_ids', () => { missing_exception_lists_count: 0, missing_rules: [], missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], }, }); }); @@ -137,7 +148,9 @@ describe('get_export_by_object_ids', () => { exceptionsClient, clients.savedObjectsClient, objects, - logger + logger, + exporterMock, + requestMock ); const details = getOutputDetailsSampleWithExceptions({ missingRules: [{ rule_id: 'rule-1' }], @@ -147,6 +160,180 @@ describe('get_export_by_object_ids', () => { rulesNdjson: '', exportDetails: getSampleDetailsAsNdjson(details), exceptionLists: '', + actionConnectors: '', + }); + }); + + test('it will export with rule and action connectors', async () => { + const rulesClient = rulesClientMock.create(); + const result = getFindResultWithSingleHit(); + const alert = { + ...getRuleMock(getQueryRuleParams()), + actions: [ + { + group: 'default', + id: '123', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + actionTypeId: '.slack', + }, + ], + }; + + alert.params = { + ...alert.params, + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + threat: getThreatMock(), + meta: { someMeta: 'someField' }, + timelineId: 'some-timeline-id', + timelineTitle: 'some-timeline-title', + }; + result.data = [alert]; + rulesClient.find.mockResolvedValue(result); + let eventCount = 0; + const readable = new Readable({ + objectMode: true, + read() { + if (eventCount === 0) { + eventCount += 1; + return this.push({ + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + type: 'action', + updated_at: '2023-01-11T11:30:31.683Z', + created_at: '2023-01-11T11:30:31.683Z', + version: 'WzE2MDYsMV0=', + attributes: { + actionTypeId: '.slack', + name: 'slack', + isMissingSecrets: true, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }); + } + if (eventCount === 1) { + eventCount += 1; + return this.push({ + exportedCount: 1, + missingRefCount: 0, + missingReferences: [], + excludedObjectsCount: 0, + excludedObjects: [], + }); + } + return this.push(null); + }, + }); + const objects = [{ rule_id: 'rule-1' }]; + const exporterMockWithConnector = { + exportByObjects: () => jest.fn().mockReturnValueOnce(readable), + + exportByTypes: jest.fn(), + }; + const exports = await getExportByObjectIds( + rulesClient, + exceptionsClient, + clients.savedObjectsClient, + objects, + logger, + exporterMockWithConnector as never, + requestMock + ); + const rulesJson = JSON.parse(exports.rulesNdjson); + const detailsJson = JSON.parse(exports.exportDetails); + const actionConnectorsJSON = JSON.parse(exports.actionConnectors); + expect(rulesJson).toEqual({ + author: ['Elastic'], + actions: [ + { + group: 'default', + id: '123', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + action_type_id: '.slack', + }, + ], + building_block_type: 'default', + created_at: '2019-12-13T16:40:33.400Z', + updated_at: '2019-12-13T16:40:33.400Z', + created_by: 'elastic', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + from: 'now-6m', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + rule_id: 'rule-1', + language: 'kuery', + license: 'Elastic License', + output_index: '.siem-signals', + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + name: 'Detect Root/Admin Users', + query: 'user.name: root or user.name: admin', + references: ['http://example.com', 'https://example.com'], + related_integrations: [], + required_fields: [], + setup: '', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + meta: { someMeta: 'someField' }, + severity: 'high', + severity_mapping: [], + updated_by: 'elastic', + tags: [], + to: 'now', + type: 'query', + threat: getThreatMock(), + throttle: 'rule', + note: '# Investigative notes', + version: 1, + exceptions_list: getListArrayMock(), + }); + expect(detailsJson).toEqual({ + exported_exception_list_count: 0, + exported_exception_list_item_count: 0, + exported_count: 2, + exported_rules_count: 1, + missing_exception_list_item_count: 0, + missing_exception_list_items: [], + missing_exception_lists: [], + missing_exception_lists_count: 0, + missing_rules: [], + missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 1, + missing_action_connection_count: 0, + missing_action_connections: [], + }); + expect(actionConnectorsJSON).toEqual({ + attributes: { + actionTypeId: '.slack', + config: {}, + isMissingSecrets: true, + name: 'slack', + secrets: {}, + }, + coreMigrationVersion: '8.7.0', + created_at: '2023-01-11T11:30:31.683Z', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + migrationVersion: { + action: '8.3.0', + }, + references: [], + type: 'action', + updated_at: '2023-01-11T11:30:31.683Z', + version: 'WzE2MDYsMV0=', }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts index e15ab98e0d0ca..90938420e64b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.ts @@ -6,9 +6,10 @@ */ import { chunk } from 'lodash'; + import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import type { Logger } from '@kbn/core/server'; +import type { ISavedObjectsExporter, KibanaRequest, Logger } from '@kbn/core/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; import type { RulesClient, RuleExecutorServices } from '@kbn/alerting-plugin/server'; @@ -17,6 +18,7 @@ import { getExportDetailsNdjson } from './get_export_details_ndjson'; import { isAlertType } from '../../../rule_schema'; import { findRules } from '../search/find_rules'; import { getRuleExceptionsForExport } from './get_export_rule_exceptions'; +import { getRuleActionConnectorsForExport } from './get_export_rule_action_connectors'; // eslint-disable-next-line no-restricted-imports import { legacyGetBulkRuleActionsSavedObject } from '../../../rule_actions_legacy'; @@ -44,11 +46,14 @@ export const getExportByObjectIds = async ( exceptionsClient: ExceptionListClient | undefined, savedObjectsClient: RuleExecutorServices['savedObjectsClient'], objects: Array<{ rule_id: string }>, - logger: Logger + logger: Logger, + actionsExporter: ISavedObjectsExporter, + request: KibanaRequest ): Promise<{ rulesNdjson: string; exportDetails: string; exceptionLists: string | null; + actionConnectors: string; }> => { const rulesAndErrors = await getRulesFromObjects( rulesClient, @@ -56,23 +61,33 @@ export const getExportByObjectIds = async ( objects, logger ); + const { rules, missingRules } = rulesAndErrors; // Retrieve exceptions - const exceptions = rulesAndErrors.rules.flatMap((rule) => rule.exceptions_list ?? []); + const exceptions = rules.flatMap((rule) => rule.exceptions_list ?? []); const { exportData: exceptionLists, exportDetails: exceptionDetails } = await getRuleExceptionsForExport(exceptions, exceptionsClient); - const rulesNdjson = transformDataToNdjson(rulesAndErrors.rules); + // Retrieve Action-Connectors + const { actionConnectors, actionConnectorDetails } = await getRuleActionConnectorsForExport( + rules, + actionsExporter, + request + ); + + const rulesNdjson = transformDataToNdjson(rules); const exportDetails = getExportDetailsNdjson( - rulesAndErrors.rules, - rulesAndErrors.missingRules, - exceptionDetails + rules, + missingRules, + exceptionDetails, + actionConnectorDetails ); return { rulesNdjson, exportDetails, exceptionLists, + actionConnectors, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_details_ndjson.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_details_ndjson.ts index 465c4b53b1e51..9efbe665812c9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_details_ndjson.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_details_ndjson.ts @@ -8,23 +8,29 @@ import type { ExportExceptionDetails } from '@kbn/securitysolution-io-ts-list-types'; import type { ExportRulesDetails } from '../../../../../../common/detection_engine/rule_management'; import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; +import type { DefaultActionConnectorDetails } from './get_export_rule_action_connectors'; export const getExportDetailsNdjson = ( rules: RuleResponse[], missingRules: Array<{ rule_id: string }> = [], - exceptionDetails?: ExportExceptionDetails + exceptionDetails?: ExportExceptionDetails, + actionConnectorDetails?: DefaultActionConnectorDetails ): string => { + let exportedCount = rules.length; + if (actionConnectorDetails != null) + exportedCount += actionConnectorDetails.exported_action_connector_count; + if (exceptionDetails != null) + exportedCount += + exceptionDetails.exported_exception_list_count + + exceptionDetails.exported_exception_list_item_count; + const stringified: ExportRulesDetails = { - exported_count: - exceptionDetails == null - ? rules.length - : rules.length + - exceptionDetails.exported_exception_list_count + - exceptionDetails.exported_exception_list_item_count, + exported_count: exportedCount, exported_rules_count: rules.length, missing_rules: missingRules, missing_rules_count: missingRules.length, ...exceptionDetails, + ...actionConnectorDetails, }; return `${JSON.stringify(stringified)}\n`; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_rule_action_connectors.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_rule_action_connectors.ts new file mode 100644 index 0000000000000..04472ff6b4598 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_rule_action_connectors.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { KibanaRequest } from '@kbn/core-http-server'; +import type { SavedObjectTypeIdTuple } from '@kbn/core-saved-objects-common'; +import type { + SavedObjectsExportResultDetails, + ISavedObjectsExporter, + SavedObjectsExportExcludedObject, + SavedObject, +} from '@kbn/core-saved-objects-server'; +import { createConcatStream, createMapStream, createPromiseFromStreams } from '@kbn/utils'; +import type { RuleResponse } from '../../../../../../common/detection_engine/rule_schema'; + +export interface DefaultActionConnectorDetails { + exported_action_connector_count: number; + missing_action_connection_count: number; + missing_action_connections: SavedObjectTypeIdTuple[]; + excluded_action_connection_count: number; + excluded_action_connections: SavedObjectsExportExcludedObject[]; +} + +const defaultActionConnectorDetails: DefaultActionConnectorDetails = { + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], + excluded_action_connection_count: 0, + excluded_action_connections: [], +}; + +const mapExportedActionConnectorsDetailsToDefault = ( + exportDetails: SavedObjectsExportResultDetails +): DefaultActionConnectorDetails => { + return { + exported_action_connector_count: exportDetails.exportedCount, + missing_action_connection_count: exportDetails.missingRefCount, + missing_action_connections: exportDetails.missingReferences, + excluded_action_connection_count: exportDetails.excludedObjectsCount, + excluded_action_connections: exportDetails.excludedObjects, + }; +}; + +export const getRuleActionConnectorsForExport = async ( + rules: RuleResponse[], + actionsExporter: ISavedObjectsExporter, + request: KibanaRequest +) => { + const exportedActionConnectors: { + actionConnectors: string; + actionConnectorDetails: DefaultActionConnectorDetails; + } = { + actionConnectors: '', + actionConnectorDetails: defaultActionConnectorDetails, + }; + + const actionsIds = [...new Set(rules.flatMap((rule) => rule.actions.map(({ id }) => id)))]; + + if (!actionsIds.length) return exportedActionConnectors; + + const getActionsByObjectsParam = actionsIds.map((id) => ({ type: 'action', id })); + const actionDetails = await actionsExporter.exportByObjects({ + objects: getActionsByObjectsParam, + request, + }); + + if (!actionDetails) { + exportedActionConnectors.actionConnectorDetails = { + exported_action_connector_count: 0, + missing_action_connection_count: actionsIds.length, + missing_action_connections: [], // TODO: check how to generate SO + excluded_action_connection_count: 0, + excluded_action_connections: [], + }; + return exportedActionConnectors; + } + + const actionsConnectorsToExport: SavedObject[] = await createPromiseFromStreams([ + actionDetails, + createMapStream((obj: SavedObject | SavedObjectsExportResultDetails) => { + if ('exportedCount' in obj) + exportedActionConnectors.actionConnectorDetails = + mapExportedActionConnectorsDetailsToDefault(obj); + else return JSON.stringify(obj); + }), + createConcatStream([]), + ]); + exportedActionConnectors.actionConnectors = actionsConnectorsToExport.join('\n'); + return exportedActionConnectors; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts new file mode 100644 index 0000000000000..455274006297e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.test.ts @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client.mock'; +import { + getImportRulesSchemaMock, + webHookConnector, +} from '../../../../../../../common/detection_engine/rule_management/model/import/rule_to_import.mock'; +import { importRuleActionConnectors } from './import_rule_action_connectors'; +import { coreMock } from '@kbn/core/server/mocks'; + +const rules = [ + { + ...getImportRulesSchemaMock(), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + action_type_id: '.webhook', + params: {}, + }, + ], + }, +]; +const rulesWithoutActions = [ + { + ...getImportRulesSchemaMock(), + actions: [], + }, +]; +const actionConnectors = [webHookConnector]; +const actionsClient = actionsClientMock.create(); +actionsClient.getAll.mockResolvedValue([]); +const core = coreMock.createRequestHandlerContext(); + +describe('checkRuleExceptionReferences', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('should show an error message when the user has a Read Actions permission and stops the importing ', async () => { + const newCore = coreMock.createRequestHandlerContext(); + const error = { + output: { payload: { message: 'Unable to bulk_create action' }, statusCode: 403 }, + }; + newCore.savedObjects.getImporter = jest.fn().mockReturnValueOnce({ + import: jest.fn().mockImplementation(() => { + throw error; + }), + }); + const actionsImporter2 = newCore.savedObjects.getImporter; + + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter2() as never, + rules, + overwrite: false, + }); + + expect(res).toEqual({ + success: false, + successCount: 0, + errors: [ + { + error: { + message: + 'You may not have actions privileges required to import rules with actions: Unable to bulk_create action', + status_code: 403, + }, + rule_id: '(unknown id)', + }, + ], + warnings: [], + }); + }); + + it('should return import 1 connector successfully', async () => { + core.savedObjects.getImporter = jest.fn().mockReturnValueOnce({ + import: jest.fn().mockResolvedValue({ + success: true, + successCount: 1, + successResults: [], + errors: [], + warnings: [], + }), + }); + const actionsImporter = core.savedObjects.getImporter; + + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter() as never, + rules, + overwrite: false, + }); + + expect(res).toEqual({ + success: true, + successCount: 1, + successResults: [], + errors: [], + warnings: [], + }); + }); + it('should return import 1 connector successfully only if id is duplicated', async () => { + core.savedObjects.getImporter = jest.fn().mockReturnValueOnce({ + import: jest.fn().mockResolvedValue({ + success: true, + successCount: 1, + successResults: [], + errors: [], + warnings: [], + }), + }); + const actionsImporter = core.savedObjects.getImporter; + + const ruleWith2Connectors = [ + { + ...getImportRulesSchemaMock(), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + action_type_id: '.slack', + }, + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + }, + action_type_id: '.slack', + }, + ], + }, + ]; + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter() as never, + rules: ruleWith2Connectors, + overwrite: false, + }); + + expect(res).toEqual({ + success: true, + successCount: 1, + successResults: [], + errors: [], + warnings: [], + }); + }); + + it('should show an error message when the user has an old imported rule with a missing connector data', async () => { + const actionsImporter = core.savedObjects.getImporter; + + const res = await importRuleActionConnectors({ + actionConnectors: [], + actionsClient, + actionsImporter: actionsImporter() as never, + rules, + overwrite: false, + }); + + expect(res).toEqual({ + success: false, + successCount: 0, + errors: [ + { + error: { + message: + '1 connector is missing. Connector id missing is: cabc78e0-9031-11ed-b076-53cc4d57aaf1', + status_code: 404, + }, + rule_id: 'rule-1', + }, + ], + warnings: [], + }); + }); + it('should show an error message when the user has an old imported rule with a 2 missing connectors data', async () => { + const actionsImporter = core.savedObjects.getImporter; + + const res = await importRuleActionConnectors({ + actionConnectors: [], + actionsClient, + actionsImporter: actionsImporter() as never, + rules: [ + { + ...getImportRulesSchemaMock(), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + action_type_id: '.webhook', + params: {}, + }, + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf2', + action_type_id: '.webhook', + params: {}, + }, + ], + }, + ], + overwrite: false, + }); + + expect(res).toEqual({ + success: false, + successCount: 0, + errors: [ + { + error: { + message: + '2 connectors are missing. Connector ids missing are: cabc78e0-9031-11ed-b076-53cc4d57aaf1, cabc78e0-9031-11ed-b076-53cc4d57aaf2', + status_code: 404, + }, + rule_id: 'rule-1', + }, + ], + warnings: [], + }); + }); + it('should show an error message when the user has 2 imported rules with a 2 missing connectors data', async () => { + const actionsImporter = core.savedObjects.getImporter; + + const res = await importRuleActionConnectors({ + actionConnectors: [], + actionsClient, + actionsImporter: actionsImporter() as never, + rules: [ + { + ...getImportRulesSchemaMock(), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + action_type_id: '.webhook', + params: {}, + }, + ], + }, + { + ...getImportRulesSchemaMock('rule-2'), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf2', + action_type_id: '.webhook', + params: {}, + }, + ], + }, + ], + overwrite: false, + }); + + expect(res).toEqual({ + success: false, + successCount: 0, + errors: [ + { + error: { + message: + '2 connectors are missing. Connector ids missing are: cabc78e0-9031-11ed-b076-53cc4d57aaf1, cabc78e0-9031-11ed-b076-53cc4d57aaf2', + status_code: 404, + }, + rule_id: 'rule-1,rule-2', + }, + ], + warnings: [], + }); + }); + + it('should skip importing the action-connectors if the actions array is empty, even if the user has exported-connectors in the file', async () => { + core.savedObjects.getImporter = jest.fn().mockReturnValue({ + import: jest.fn().mockResolvedValue({ + success: true, + successCount: 2, + successResults: [], + errors: [], + warnings: [], + }), + }); + const actionsImporter2 = core.savedObjects.getImporter; + const actionsImporter2Import = actionsImporter2().import; + + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter2Import as never, + rules: rulesWithoutActions, + overwrite: false, + }); + + expect(res).toEqual({ + success: true, + successCount: 0, + errors: [], + warnings: [], + }); + expect(actionsImporter2Import).not.toBeCalled(); + }); + + it('should skip importing the action-connectors if all connectors have been imported/created before', async () => { + actionsClient.getAll.mockResolvedValue([ + { + actionTypeId: '.webhook', + name: 'webhook', + isPreconfigured: true, + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + referencedByCount: 1, + isDeprecated: false, + }, + ]); + const actionsImporter2 = core.savedObjects.getImporter; + const actionsImporter2Import = actionsImporter2().import; + + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter2Import as never, + rules, + overwrite: false, + }); + + expect(res).toEqual({ + success: true, + successCount: 0, + errors: [], + warnings: [], + }); + expect(actionsImporter2Import).not.toBeCalled(); + }); + + it('should not skip importing the action-connectors if all connectors have been imported/created before when overwrite is true', async () => { + core.savedObjects.getImporter = jest.fn().mockReturnValueOnce({ + import: jest.fn().mockResolvedValue({ + success: true, + successCount: 1, + successResults: [], + errors: [], + warnings: [], + }), + }); + const actionsImporter = core.savedObjects.getImporter; + + actionsClient.getAll.mockResolvedValue([ + { + actionTypeId: '.webhook', + name: 'webhook', + isPreconfigured: true, + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + referencedByCount: 1, + isDeprecated: false, + }, + ]); + + const res = await importRuleActionConnectors({ + actionConnectors, + actionsClient, + actionsImporter: actionsImporter() as never, + rules, + overwrite: true, + }); + + expect(res).toEqual({ + success: true, + successCount: 1, + errors: [], + warnings: [], + successResults: [], + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.ts new file mode 100644 index 0000000000000..3d2fc9fb7c6b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/import_rule_action_connectors.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Readable } from 'stream'; + +import type { SavedObjectsImportResponse } from '@kbn/core-saved-objects-common'; +import type { SavedObject } from '@kbn/core-saved-objects-server'; + +import type { WarningSchema } from '../../../../../../../common/detection_engine/schemas/response'; +import { + checkIfActionsHaveMissingConnectors, + filterExistingActionConnectors, + getActionConnectorRules, + handleActionsHaveNoConnectors, + mapSOErrorToRuleError, + returnErroredImportResult, +} from './utils'; +import type { ImportRuleActionConnectorsParams, ImportRuleActionConnectorsResult } from './types'; + +export const importRuleActionConnectors = async ({ + actionConnectors, + actionsClient, + actionsImporter, + rules, + overwrite, +}: ImportRuleActionConnectorsParams): Promise => { + try { + const actionConnectorRules = getActionConnectorRules(rules); + const actionsIds: string[] = Object.keys(actionConnectorRules); + + if (!actionsIds.length) + return { + success: true, + errors: [], + successCount: 0, + warnings: [], + }; + + if (overwrite && !actionConnectors.length) + return handleActionsHaveNoConnectors(actionsIds, actionConnectorRules); + + let actionConnectorsToImport: SavedObject[] = actionConnectors; + + if (!overwrite) { + const newIdsToAdd = await filterExistingActionConnectors(actionsClient, actionsIds); + + const foundMissingConnectors = checkIfActionsHaveMissingConnectors( + actionConnectors, + newIdsToAdd, + actionConnectorRules + ); + if (foundMissingConnectors) return foundMissingConnectors; + // filter out existing connectors + actionConnectorsToImport = actionConnectors.filter(({ id }) => newIdsToAdd.includes(id)); + } + if (!actionConnectorsToImport.length) + return { + success: true, + errors: [], + successCount: 0, + warnings: [], + }; + + const readStream = Readable.from(actionConnectorsToImport); + const { success, successCount, successResults, warnings, errors }: SavedObjectsImportResponse = + await actionsImporter.import({ + readStream, + overwrite, + createNewCopies: false, + }); + return { + success, + successCount, + successResults, + errors: errors ? mapSOErrorToRuleError(errors) : [], + warnings: (warnings as WarningSchema[]) || [], + }; + } catch (error) { + return returnErroredImportResult(error); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/types.ts new file mode 100644 index 0000000000000..8ba6c41c42dde --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { ISavedObjectsImporter, SavedObject } from '@kbn/core-saved-objects-server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { + SavedObjectsImportFailure, + SavedObjectsImportSuccess, +} from '@kbn/core-saved-objects-common'; +import type { RuleToImport } from '../../../../../../../common/detection_engine/rule_management'; +import type { WarningSchema } from '../../../../../../../common/detection_engine/schemas/response'; +import type { BulkError } from '../../../../routes/utils'; + +export interface ImportRuleActionConnectorsResult { + success: boolean; + successCount: number; + successResults?: SavedObjectsImportSuccess[]; + errors: BulkError[] | []; + warnings: WarningSchema[] | []; +} + +export interface ImportRuleActionConnectorsParams { + actionConnectors: SavedObject[]; + actionsClient: ActionsClient; + actionsImporter: ISavedObjectsImporter; + rules: Array; + overwrite: boolean; +} + +export interface SOError { + output: { statusCode: number; payload: { message: string } }; +} + +export interface ConflictError { + type: string; +} + +export type ErrorType = SOError | ConflictError | SavedObjectsImportFailure | Error; +export interface ActionRules { + [actionsIds: string]: string[]; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/utils/index.ts new file mode 100644 index 0000000000000..6d9b3b9da9e6d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/action_connectors/utils/index.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { pick } from 'lodash'; +import type { SavedObjectsImportFailure } from '@kbn/core-saved-objects-common'; +import type { SavedObject } from '@kbn/core-saved-objects-server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { BulkError } from '../../../../../routes/utils'; +import { createBulkErrorObject } from '../../../../../routes/utils'; +import type { RuleToImport } from '../../../../../../../../common/detection_engine/rule_management'; +import type { + ActionRules, + ConflictError, + ErrorType, + ImportRuleActionConnectorsResult, + SOError, +} from '../types'; + +export const returnErroredImportResult = (error: ErrorType): ImportRuleActionConnectorsResult => ({ + success: false, + errors: [handleActionConnectorsErrors(error)], + successCount: 0, + warnings: [], +}); + +export const handleActionsHaveNoConnectors = ( + actionsIds: string[], + actionConnectorRules: ActionRules +): ImportRuleActionConnectorsResult => { + const ruleIds: string = [...new Set(Object.values(actionConnectorRules).flat())].join(); + + if (actionsIds && actionsIds.length) { + const errors: BulkError[] = []; + const errorMessage = + actionsIds.length > 1 + ? 'connectors are missing. Connector ids missing are:' + : 'connector is missing. Connector id missing is:'; + errors.push( + createBulkErrorObject({ + statusCode: 404, + message: `${actionsIds.length} ${errorMessage} ${actionsIds.join(', ')}`, + ruleId: ruleIds, + }) + ); + return { + success: false, + errors, + successCount: 0, + warnings: [], + }; + } + return { + success: true, + errors: [], + successCount: 0, + warnings: [], + }; +}; + +export const handleActionConnectorsErrors = (error: ErrorType, id?: string): BulkError => { + let statusCode: number | null = null; + let message: string = ''; + if ('output' in error) { + statusCode = (error as SOError).output.statusCode; + message = (error as SOError).output.payload?.message; + } + switch (statusCode) { + case null: + return createBulkErrorObject({ + statusCode: 500, + message: + (error as ConflictError)?.type === 'conflict' + ? 'There is a conflict' + : (error as Error).message + ? (error as Error).message + : '', + }); + + case 403: + return createBulkErrorObject({ + id, + statusCode, + message: `You may not have actions privileges required to import rules with actions: ${message}`, + }); + + default: + return createBulkErrorObject({ + id, + statusCode, + message, + }); + } +}; + +export const mapSOErrorToRuleError = (errors: SavedObjectsImportFailure[]): BulkError[] => { + return errors.map(({ id, error }) => handleActionConnectorsErrors(error, id)); +}; + +export const filterExistingActionConnectors = async ( + actionsClient: ActionsClient, + actionsIds: string[] +) => { + const storedConnectors = await actionsClient.getAll(); + const storedActionIds: string[] = storedConnectors.map(({ id }) => id); + return actionsIds.filter((id) => !storedActionIds.includes(id)); +}; +export const getActionConnectorRules = (rules: Array) => + rules.reduce((acc: { [actionsIds: string]: string[] }, rule) => { + if (rule instanceof Error) return acc; + rule.actions?.forEach(({ id }) => (acc[id] = [...(acc[id] || []), rule.rule_id])); + return acc; + }, {}); +export const checkIfActionsHaveMissingConnectors = ( + actionConnectors: SavedObject[], + newIdsToAdd: string[], + actionConnectorRules: ActionRules +) => { + // if new action-connectors don't have exported connectors will fail with missing connectors + if (actionConnectors.length < newIdsToAdd.length) { + const actionConnectorsIds = actionConnectors.map(({ id }) => id); + const missingActionConnector = newIdsToAdd.filter((id) => !actionConnectorsIds.includes(id)); + const missingActionRules = pick(actionConnectorRules, [...missingActionConnector]); + return handleActionsHaveNoConnectors(missingActionConnector, missingActionRules); + } + return null; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts index e325496225d9f..4414b6063c473 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts @@ -24,6 +24,7 @@ import type { ImportExceptionsListSchema, } from '@kbn/securitysolution-io-ts-list-types'; +import type { SavedObject } from '@kbn/core-saved-objects-server'; import { RuleToImport, validateRuleToImport, @@ -41,7 +42,9 @@ export const validateRulesStream = (): Transform => { return createMapStream<{ exceptions: Array; rules: Array; + actionConnectors: SavedObject[]; }>((items) => ({ + actionConnectors: items.actionConnectors, exceptions: items.exceptions, rules: validateRules(items.rules), })); @@ -80,10 +83,14 @@ export const sortImports = (): Transform => { return createReduceStream<{ exceptions: Array; rules: Array; + actionConnectors: SavedObject[]; }>( (acc, importItem) => { if (has('list_id', importItem) || has('item_id', importItem) || has('entries', importItem)) { return { ...acc, exceptions: [...acc.exceptions, importItem] }; + } + if (has('attributes', importItem)) { + return { ...acc, actionConnectors: [...acc.actionConnectors, importItem] }; } else { return { ...acc, rules: [...acc.rules, importItem] }; } @@ -91,6 +98,7 @@ export const sortImports = (): Transform => { { exceptions: [], rules: [], + actionConnectors: [], } ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts index 694bc916fefe3..5a8d6f9a34e87 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/import_rules_utils.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; import type { ImportExceptionsListSchema, ImportExceptionListItemSchema, @@ -31,6 +31,7 @@ export type PromiseFromStreams = RuleToImport | Error; export interface RuleExceptionsPromiseFromStreams { rules: PromiseFromStreams[]; exceptions: Array; + actionConnectors: SavedObject[]; } /** @@ -60,6 +61,7 @@ export const importRules = async ({ exceptionsClient, spaceId, existingLists, + allowMissingConnectorSecrets, }: { ruleChunks: PromiseFromStreams[][]; rulesResponseAcc: ImportRuleResponse[]; @@ -70,6 +72,7 @@ export const importRules = async ({ exceptionsClient: ExceptionListClient | undefined; spaceId: string; existingLists: Record; + allowMissingConnectorSecrets?: boolean; }) => { let importRuleResponse: ImportRuleResponse[] = [...rulesResponseAcc]; @@ -118,6 +121,7 @@ export const importRules = async ({ ...parsedRule, exceptions_list: [...exceptions], }, + allowMissingConnectorSecrets, }); resolve({ rule_id: parsedRule.rule_id, @@ -136,6 +140,7 @@ export const importRules = async ({ ...parsedRule, exceptions_list: [...exceptions], }, + allowMissingConnectorSecrets, }); resolve({ rule_id: parsedRule.rule_id, diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 4522a35f793ec..531fd7c0a6f38 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -133,6 +133,8 @@ "@kbn/cypress-config", "@kbn/controls-plugin", "@kbn/shared-ux-utility", + "@kbn/core-saved-objects-common", + "@kbn/core-saved-objects-import-export-server-mocks", "@kbn/user-profile-components", "@kbn/guided-onboarding", "@kbn/securitysolution-ecs", diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts b/x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts index 833b9ccb9ee1f..2d1ddd342a549 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/export_rules.ts @@ -87,6 +87,11 @@ export default ({ getService }: FtrProviderContext): void => { missing_exception_lists_count: 0, missing_rules: [], missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], }); }); diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/import_rules.ts b/x-pack/test/detection_engine_api_integration/basic/tests/import_rules.ts index d32e93b481547..2630174529b75 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/import_rules.ts @@ -73,6 +73,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -147,6 +151,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -170,6 +178,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -227,6 +239,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -245,6 +261,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -277,6 +297,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -301,6 +325,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -365,6 +393,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -404,6 +436,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts index b30e16681d488..60af78ec6cff9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts @@ -88,6 +88,11 @@ export default ({ getService }: FtrProviderContext): void => { missing_exception_lists_count: 0, missing_rules: [], missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts index 48c4408e8555e..169fe074604fa 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts @@ -135,9 +135,14 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); - it('should successfully import rules with actions when user has "read" actions privileges', async () => { + + it('should not import rules with actions when user has "read" actions privileges', async () => { // create a new action const { body: hookAction } = await supertest .post('/api/actions/action') @@ -149,27 +154,71 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: hookAction.id, + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', action_type_id: hookAction.actionTypeId, params: {}, }, ], }; + const ruleWithConnector = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf1', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: true, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; + const { body } = await supertestWithoutAuth .post(`${DETECTION_ENGINE_RULES_URL}/_import`) .auth(ROLES.hunter, 'changeme') .set('kbn-xsrf', 'true') - .attach('file', ruleToNdjson(simpleRule), 'rules.ndjson') + .attach( + 'file', + Buffer.from(toNdJsonString([simpleRule, ruleWithConnector])), + 'rules.ndjson' + ) .expect(200); expect(body).to.eql({ - errors: [], - success: true, - success_count: 1, + errors: [ + { + error: { + message: + 'You may not have actions privileges required to import rules with actions: Unable to bulk_create action', + status_code: 403, + }, + rule_id: '(unknown id)', + }, + ], + success: false, + success_count: 0, rules_count: 1, exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: false, + action_connectors_success_count: 0, + action_connectors_errors: [ + { + error: { + message: + 'You may not have actions privileges required to import rules with actions: Unable to bulk_create action', + status_code: 403, + }, + rule_id: '(unknown id)', + }, + ], + action_connectors_warnings: [], }); }); it('should not import rules with actions when a user has no actions privileges', async () => { @@ -184,17 +233,39 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: hookAction.id, + id: 'cabc78e0-9031-11ed-b076-53cc4d57axy1', action_type_id: hookAction.actionTypeId, params: {}, }, ], }; + const ruleWithConnector = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57axy1', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: true, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; + const { body } = await supertestWithoutAuth .post(`${DETECTION_ENGINE_RULES_URL}/_import`) .auth(ROLES.hunter_no_actions, 'changeme') .set('kbn-xsrf', 'true') - .attach('file', ruleToNdjson(simpleRule), 'rules.ndjson') + .attach( + 'file', + Buffer.from(toNdJsonString([simpleRule, ruleWithConnector])), + 'rules.ndjson' + ) .expect(200); expect(body).to.eql({ success: false, @@ -208,18 +279,24 @@ export default ({ getService }: FtrProviderContext): void => { }, rule_id: '(unknown id)', }, - { - error: { - message: `1 connector is missing. Connector id missing is: ${hookAction.id}`, - status_code: 404, - }, - rule_id: 'rule-1', - }, ], rules_count: 1, exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: false, + action_connectors_success_count: 0, + action_connectors_errors: [ + { + error: { + message: + 'You may not have actions privileges required to import rules with actions: Unauthorized to get actions', + status_code: 403, + }, + rule_id: '(unknown id)', + }, + ], + action_connectors_warnings: [], }); }); }); @@ -367,6 +444,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -404,6 +485,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -430,6 +515,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -448,6 +537,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -480,6 +573,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -504,6 +601,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -568,6 +669,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -607,6 +712,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -688,6 +797,78 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: false, + action_connectors_success_count: 0, + action_connectors_warnings: [], + action_connectors_errors: [ + { + rule_id: 'rule-1', + error: { + status_code: 404, + message: '1 connector is missing. Connector id missing is: 123', + }, + }, + ], + }); + }); + it('should give single connector warning back if we have a single connector missing secret', async () => { + const simpleRule: ReturnType = { + ...getSimpleRule('rule-1'), + actions: [ + { + group: 'default', + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf9', + action_type_id: '.webhook', + params: {}, + }, + ], + }; + const ruleWithConnector = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57aaf9', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: true, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .attach( + 'file', + Buffer.from(toNdJsonString([simpleRule, ruleWithConnector])), + 'rules.ndjson' + ) + .expect(200); + + expect(body).to.eql({ + success: true, + success_count: 1, + rules_count: 1, + errors: [], + exceptions_errors: [], + exceptions_success: true, + exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 1, + action_connectors_warnings: [ + { + actionPath: '/app/management/insightsAndAlerting/triggersActionsConnectors', + buttonLabel: 'Go to connectors', + message: '1 connector has sensitive information that require updates.', + type: 'action_required', + }, + ], + action_connectors_errors: [], }); }); @@ -709,6 +890,7 @@ export default ({ getService }: FtrProviderContext): void => { }, ], }; + const { body } = await supertest .post(`${DETECTION_ENGINE_RULES_URL}/_import`) .set('kbn-xsrf', 'true') @@ -722,10 +904,14 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); - it('should be able to import 2 rules with action connectors that exist', async () => { + it('should be able to import 2 rules with action connectors', async () => { // create a new action const { body: hookAction } = await supertest .post('/api/actions/action') @@ -738,7 +924,7 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: hookAction.id, + id: 'cabc78e0-9031-11ed-b076-53cc4d57abc6', action_type_id: hookAction.actionTypeId, params: {}, }, @@ -750,18 +936,57 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: hookAction.id, - action_type_id: hookAction.actionTypeId, + id: 'f4e74ab0-9e59-11ed-a3db-f9134a9ce951', + action_type_id: '.index', params: {}, }, ], }; + + const connector1 = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57abc6', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: false, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; + const connector2 = { + id: 'f4e74ab0-9e59-11ed-a3db-f9134a9ce951', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.index', + name: 'index', + isMissingSecrets: false, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; const rule1String = JSON.stringify(rule1); const rule2String = JSON.stringify(rule2); - const buffer = Buffer.from(`${rule1String}\n${rule2String}\n`); + const connector12String = JSON.stringify(connector1); + const connector22String = JSON.stringify(connector2); + const buffer = Buffer.from( + `${rule1String}\n${rule2String}\n${connector12String}\n${connector22String}\n` + ); const { body } = await supertest - .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .post(`${DETECTION_ENGINE_RULES_URL}/_import?overwrite=true`) .set('kbn-xsrf', 'true') .attach('file', buffer, 'rules.ndjson') .expect(200); @@ -774,6 +999,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 2, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -790,7 +1019,7 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: hookAction.id, + id: 'cabc78e0-9031-11ed-b076-53cc4d57aayo', action_type_id: hookAction.actionTypeId, params: {}, }, @@ -802,15 +1031,36 @@ export default ({ getService }: FtrProviderContext): void => { actions: [ { group: 'default', - id: '123', // <-- This does not exist - action_type_id: hookAction.actionTypeId, + id: 'cabc78e0-9031-11ed-b076-53cc4d57aa22', // <-- This does not exist + action_type_id: '.index', params: {}, }, ], }; + + const connector = { + id: 'cabc78e0-9031-11ed-b076-53cc4d57aayo', + type: 'action', + updated_at: '2023-01-25T14:35:52.852Z', + created_at: '2023-01-25T14:35:52.852Z', + version: 'WzUxNTksMV0=', + attributes: { + actionTypeId: '.webhook', + name: 'webhook', + isMissingSecrets: false, + config: {}, + secrets: {}, + }, + references: [], + migrationVersion: { action: '8.3.0' }, + coreMigrationVersion: '8.7.0', + }; + const rule1String = JSON.stringify(rule1); const rule2String = JSON.stringify(rule2); - const buffer = Buffer.from(`${rule1String}\n${rule2String}\n`); + const connector2String = JSON.stringify(connector); + + const buffer = Buffer.from(`${rule1String}\n${rule2String}\n${connector2String}\n`); const { body } = await supertest .post(`${DETECTION_ENGINE_RULES_URL}/_import`) @@ -820,20 +1070,34 @@ export default ({ getService }: FtrProviderContext): void => { expect(body).to.eql({ success: false, - success_count: 1, + success_count: 0, rules_count: 2, errors: [ { rule_id: 'rule-2', error: { status_code: 404, - message: '1 connector is missing. Connector id missing is: 123', + message: + '1 connector is missing. Connector id missing is: cabc78e0-9031-11ed-b076-53cc4d57aa22', }, }, ], exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: false, + action_connectors_success_count: 0, + action_connectors_errors: [ + { + error: { + status_code: 404, + message: + '1 connector is missing. Connector id missing is: cabc78e0-9031-11ed-b076-53cc4d57aa22', + }, + rule_id: 'rule-2', + }, + ], + action_connectors_warnings: [], }); }); @@ -974,6 +1238,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -1045,6 +1313,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -1144,6 +1416,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -1274,6 +1550,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts index 06e55fdfdda1b..13f1f99fc3b23 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts @@ -106,6 +106,100 @@ export default ({ getService }: FtrProviderContext): void => { missing_exception_lists_count: 0, missing_rules: [], missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], + }); + }); + it('should export rules with actions connectors', async () => { + // create new actions + const webHookAction = await createWebHookConnector(); + + const defaultRuleAction = { + id: webHookAction.id, + action_type_id: '.webhook', + group: 'default', + params: { + body: '{"test":"a default action"}', + }, + }; + + const ruleId = 'rule-1'; + await createRule(supertest, log, { + ...getSimpleRule(ruleId), + actions: [defaultRuleAction], + }); + const exportedConnectors = { + attributes: { + actionTypeId: '.webhook', + config: { + hasAuth: true, + method: 'post', + url: 'http://localhost', + }, + isMissingSecrets: true, + name: 'Some connector', + secrets: {}, + }, + coreMigrationVersion: '8.7.0', + id: webHookAction.id, + migrationVersion: { + action: '8.3.0', + }, + references: [], + type: 'action', + }; + + const { body } = await postBulkAction() + .send({ query: '', action: BulkActionType.export }) + .expect(200) + .expect('Content-Type', 'application/ndjson') + .expect('Content-Disposition', 'attachment; filename="rules_export.ndjson"') + .parse(binaryToString); + + const [ruleJson, connectorsJson, exportDetailsJson] = body.toString().split(/\n/); + + const rule = removeServerGeneratedProperties(JSON.parse(ruleJson)); + expect(rule).to.eql({ + ...getSimpleRuleOutput(), + throttle: 'rule', + actions: [ + { + action_type_id: '.webhook', + group: 'default', + id: webHookAction.id, + params: { + body: '{"test":"a default action"}', + }, + }, + ], + }); + const { attributes, id, type } = JSON.parse(connectorsJson); + expect(attributes.actionTypeId).to.eql(exportedConnectors.attributes.actionTypeId); + expect(id).to.eql(exportedConnectors.id); + expect(type).to.eql(exportedConnectors.type); + expect(attributes.name).to.eql(exportedConnectors.attributes.name); + expect(attributes.secrets).to.eql(exportedConnectors.attributes.secrets); + expect(attributes.isMissingSecrets).to.eql(exportedConnectors.attributes.isMissingSecrets); + const exportDetails = JSON.parse(exportDetailsJson); + expect(exportDetails).to.eql({ + exported_exception_list_count: 0, + exported_exception_list_item_count: 0, + exported_count: 2, + exported_rules_count: 1, + missing_exception_list_item_count: 0, + missing_exception_list_items: [], + missing_exception_lists: [], + missing_exception_lists_count: 0, + missing_rules: [], + missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 1, + missing_action_connection_count: 0, + missing_action_connections: [], }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts index 3ff748fa5259d..135bf49c15ff9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/import_rules.ts @@ -133,6 +133,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); it('should successfully import rules with actions when user has "read" actions privileges', async () => { @@ -168,6 +172,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); it('should not import rules with actions when a user has no actions privileges', async () => { @@ -202,7 +210,7 @@ export default ({ getService }: FtrProviderContext): void => { { error: { message: - 'You may not have actions privileges required to import rules with actions: Unauthorized to get actions', + 'You may not have actions privileges required to import rules with actions: Unable to bulk_create action', status_code: 403, }, rule_id: '(unknown id)', @@ -218,6 +226,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); }); @@ -268,6 +280,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -305,6 +321,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -331,6 +351,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -349,6 +373,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -381,6 +409,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -405,6 +437,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -469,6 +505,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -508,6 +548,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -589,6 +633,18 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [ + { + rule_id: 'rule-1', + error: { + status_code: 404, + message: '1 connector is missing. Connector id missing is: 123', + }, + }, + ], + action_connectors_warnings: [], }); }); @@ -623,6 +679,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -675,6 +735,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -735,6 +799,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -875,6 +943,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -946,6 +1018,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 0, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -1045,6 +1121,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); @@ -1175,6 +1255,10 @@ export default ({ getService }: FtrProviderContext): void => { exceptions_errors: [], exceptions_success: true, exceptions_success_count: 1, + action_connectors_success: true, + action_connectors_success_count: 0, + action_connectors_errors: [], + action_connectors_warnings: [], }); }); }); From 7ee34928adbd7e6f063d0aa7c4d3d4fb5dc856b9 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Mon, 6 Feb 2023 12:11:11 -0600 Subject: [PATCH 004/134] [ML] Fix saved search filters not behaving correctly for complex filters (#149459) ## Summary Addresses https://github.com/elastic/kibana/issues/147705 and fix an issue with ML filter bar in Data Visualizer and AIOps pape not supporting complex filters correctly in 8.7. Changes include: - Using the `getSavedSearch` utility to get the `selectedSavedSearch` instead of the SavedSearchSimpleObj (which we previously manually parsed the json search source). - This requires some additional page dependency to be injected. - Rename `currentSavedSearch` to `deprecatedSavedSearchObj` for clarity. Note that not all deprecatedSavedSearchObj has been replaced with the new `selectedSavedSearch`. Areas that still use the deprecatedSavedSearchObj: - AD job creator Potential todos: - Support for loading the saved time range & refresh interval upon restoring the saved search - Support for AggregateQuery (currently, we only support lucene and kuery query types) #150091 ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../public/application/utils/search_utils.ts | 45 +++-- .../change_point_detetion_root.tsx | 5 +- .../explain_log_rate_spikes_app_state.tsx | 8 +- .../explain_log_rate_spikes_page.tsx | 12 +- .../log_categorization_app_state.tsx | 3 +- .../log_categorization_page.tsx | 16 +- x-pack/plugins/aiops/public/hooks/use_data.ts | 27 ++- .../aiops/public/hooks/use_data_source.ts | 5 +- x-pack/plugins/data_visualizer/kibana.json | 3 +- .../common/util/field_types_utils.ts | 2 +- .../index_data_visualizer_view.tsx | 11 +- .../components/search_panel/search_panel.tsx | 2 +- .../grid_embeddable/grid_embeddable.tsx | 8 +- .../hooks/use_data_visualizer_grid_data.ts | 4 +- .../index_data_visualizer.tsx | 42 ++-- .../utils/saved_search_utils.test.ts | 183 +++++++++++++++--- .../utils/saved_search_utils.ts | 66 ++++--- x-pack/plugins/data_visualizer/tsconfig.json | 1 + x-pack/plugins/ml/kibana.json | 1 + .../aiops/change_point_detection.tsx | 2 +- .../aiops/explain_log_rate_spikes.tsx | 2 +- .../application/aiops/log_categorization.tsx | 2 +- x-pack/plugins/ml/public/application/app.tsx | 4 + .../data_recognizer/data_recognizer.d.ts | 4 +- .../ml/__mocks__/kibana_context_value.ts | 8 +- .../contexts/ml/__mocks__/saved_search.ts | 2 +- .../application/contexts/ml/ml_context.ts | 5 +- .../contexts/ml/use_current_saved_search.ts | 20 -- .../application/contexts/ml/use_ml_context.ts | 5 +- .../configuration_step_form.tsx | 8 +- .../configuration_step/use_saved_search.ts | 11 +- .../jobs/new_job/pages/job_type/page.tsx | 31 ++- .../jobs/new_job/pages/new_job/page.tsx | 4 +- .../new_job/pages/new_job/wizard_steps.tsx | 6 +- .../jobs/new_job/recognize/page.tsx | 29 ++- .../ml/public/application/routing/router.tsx | 6 + .../routing/routes/access_denied.tsx | 9 +- .../routes/aiops/change_point_detection.tsx | 15 +- .../routes/aiops/explain_log_rate_spikes.tsx | 20 +- .../routes/aiops/log_categorization.tsx | 18 +- .../analytics_job_creation.tsx | 17 +- .../analytics_job_exploration.tsx | 1 + .../analytics_jobs_list.tsx | 1 + .../data_frame_analytics/analytics_map.tsx | 1 + .../analytics_source_selection.tsx | 1 + .../routes/datavisualizer/datavisualizer.tsx | 17 +- .../routes/datavisualizer/file_based.tsx | 19 +- .../routes/datavisualizer/index_based.tsx | 18 +- .../application/routing/routes/explorer.tsx | 1 + .../application/routing/routes/jobs_list.tsx | 1 + .../routing/routes/new_job/from_lens.tsx | 13 +- .../routing/routes/new_job/from_map.tsx | 15 +- .../routes/new_job/index_or_search.tsx | 1 + .../routing/routes/new_job/job_type.tsx | 1 + .../routing/routes/new_job/recognize.tsx | 4 +- .../routing/routes/new_job/wizard.tsx | 1 + .../routing/routes/notifications.tsx | 20 +- .../application/routing/routes/overview.tsx | 20 +- .../routing/routes/settings/calendar_list.tsx | 18 +- .../routes/settings/calendar_new_edit.tsx | 18 +- .../routing/routes/settings/filter_list.tsx | 18 +- .../routes/settings/filter_list_new_edit.tsx | 18 +- .../routing/routes/settings/settings.tsx | 18 +- .../routing/routes/timeseriesexplorer.tsx | 1 + .../routes/trained_models/models_list.tsx | 1 + .../routes/trained_models/nodes_list.tsx | 1 + .../application/routing/use_resolver.test.ts | 23 ++- .../application/routing/use_resolver.ts | 27 ++- x-pack/plugins/ml/tsconfig.json | 1 + 69 files changed, 629 insertions(+), 322 deletions(-) delete mode 100644 x-pack/plugins/ml/public/application/contexts/ml/use_current_saved_search.ts diff --git a/x-pack/plugins/aiops/public/application/utils/search_utils.ts b/x-pack/plugins/aiops/public/application/utils/search_utils.ts index 3bd0ea7a96523..f8f570d475fc0 100644 --- a/x-pack/plugins/aiops/public/application/utils/search_utils.ts +++ b/x-pack/plugins/aiops/public/application/utils/search_utils.ts @@ -11,8 +11,8 @@ import { cloneDeep } from 'lodash'; import { IUiSettingsClient } from '@kbn/core/public'; import { getEsQueryConfig, SearchSource } from '@kbn/data-plugin/common'; -import { SavedSearch } from '@kbn/discover-plugin/public'; -import { FilterManager } from '@kbn/data-plugin/public'; +import type { SavedSearch } from '@kbn/discover-plugin/public'; +import { FilterManager, isQuery, mapAndFlattenFilters } from '@kbn/data-plugin/public'; import { fromKueryExpression, toElasticsearchQuery, @@ -20,6 +20,7 @@ import { buildEsQuery, Query, Filter, + AggregateQuery, } from '@kbn/es-query'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataView } from '@kbn/data-views-plugin/public'; @@ -88,14 +89,15 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObjec * Should also form a valid query if only the query or filters is provided */ export function createMergedEsQuery( - query?: Query, + query?: Query | AggregateQuery, filters?: Filter[], dataView?: DataView, uiSettings?: IUiSettingsClient ) { let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); - if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { + // FIXME: Add support for AggregateQuery type #150091 + if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); if (query.query !== '') { combinedQuery = toElasticsearchQuery(ast, dataView); @@ -127,6 +129,14 @@ export function createMergedEsQuery( return combinedQuery; } +function getSavedSearchSource(savedSearch: SavedSearch) { + return savedSearch && + 'searchSource' in savedSearch && + savedSearch?.searchSource instanceof SearchSource + ? savedSearch.searchSource + : undefined; +} + /** * Extract query data from the saved search object * with overrides from the provided query data and/or filters @@ -141,7 +151,7 @@ export function getEsQueryFromSavedSearch({ }: { dataView: DataView; uiSettings: IUiSettingsClient; - savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; + savedSearch: SavedSearch | null | undefined; query?: Query; filters?: Filter[]; filterManager?: FilterManager; @@ -151,17 +161,13 @@ export function getEsQueryFromSavedSearch({ const userQuery = query; const userFilters = filters; + const savedSearchSource = getSavedSearchSource(savedSearch); + // If saved search has a search source with nested parent // e.g. a search coming from Dashboard saved search embeddable // which already combines both the saved search's original query/filters and the Dashboard's // then no need to process any further - if ( - savedSearch && - 'searchSource' in savedSearch && - savedSearch?.searchSource instanceof SearchSource && - savedSearch.searchSource.getParent() !== undefined && - userQuery - ) { + if (savedSearchSource && savedSearchSource.getParent() !== undefined && userQuery) { // Flattened query from search source may contain a clause that narrows the time range // which might interfere with global time pickers so we need to remove const savedQuery = @@ -181,12 +187,8 @@ export function getEsQueryFromSavedSearch({ }; } - // If saved search is an json object with the original query and filter - // retrieve the parsed query and filter - const savedSearchData = getQueryFromSavedSearchObject(savedSearch); - // If no saved search available, use user's query and filters - if (!savedSearchData && userQuery) { + if (!savedSearch && userQuery) { if (filterManager && userFilters) filterManager.addFilters(userFilters); const combinedQuery = createMergedEsQuery( @@ -205,11 +207,12 @@ export function getEsQueryFromSavedSearch({ // If saved search available, merge saved search with the latest user query or filters // which might differ from extracted saved search data - if (savedSearchData) { + if (savedSearchSource) { const globalFilters = filterManager?.getGlobalFilters(); - const currentQuery = userQuery ?? savedSearchData?.query; - const currentFilters = userFilters ?? savedSearchData?.filter; - + // FIXME: Add support for AggregateQuery type #150091 + const currentQuery = userQuery ?? (savedSearchSource.getField('query') as Query); + const currentFilters = + userFilters ?? mapAndFlattenFilters(savedSearchSource.getField('filter') as Filter[]); if (filterManager) filterManager.setFilters(currentFilters); if (globalFilters) filterManager?.addFilters(globalFilters); diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detetion_root.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detetion_root.tsx index fb9dbc5156ed4..d2d275f0b2251 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detetion_root.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detetion_root.tsx @@ -11,7 +11,7 @@ import { pick } from 'lodash'; import { EuiSpacer } from '@elastic/eui'; import { DataView } from '@kbn/data-views-plugin/common'; -import { SavedSearch } from '@kbn/saved-search-plugin/public'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import { StorageContextProvider } from '@kbn/ml-local-storage'; import { UrlStateProvider } from '@kbn/ml-url-state'; import { Storage } from '@kbn/kibana-utils-plugin/public'; @@ -20,7 +20,6 @@ import { UI_SETTINGS } from '@kbn/data-plugin/common'; import { toMountPoint, wrapWithTheme } from '@kbn/kibana-react-plugin/public'; import { DataSourceContext } from '../../hooks/use_data_source'; -import { SavedSearchSavedObject } from '../../application/utils/search_utils'; import { AiopsAppContext, AiopsAppDependencies } from '../../hooks/use_aiops_app_context'; import { AIOPS_STORAGE_KEYS } from '../../types/storage'; @@ -33,7 +32,7 @@ const localStorage = new Storage(window.localStorage); export interface ChangePointDetectionAppStateProps { dataView: DataView; - savedSearch: SavedSearch | SavedSearchSavedObject | null; + savedSearch: SavedSearch | null; appDependencies: AiopsAppDependencies; } diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_app_state.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_app_state.tsx index ae67c55d87f05..f43b60d7daffa 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_app_state.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_app_state.tsx @@ -21,11 +21,7 @@ import { DatePickerContextProvider } from '@kbn/ml-date-picker'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; import { toMountPoint, wrapWithTheme } from '@kbn/kibana-react-plugin/public'; -import { - SEARCH_QUERY_LANGUAGE, - SearchQueryLanguage, - SavedSearchSavedObject, -} from '../../application/utils/search_utils'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../application/utils/search_utils'; import type { AiopsAppDependencies } from '../../hooks/use_aiops_app_context'; import { AiopsAppContext } from '../../hooks/use_aiops_app_context'; import { DataSourceContext } from '../../hooks/use_data_source'; @@ -41,7 +37,7 @@ export interface ExplainLogRateSpikesAppStateProps { /** The data view to analyze. */ dataView: DataView; /** The saved search to analyze. */ - savedSearch: SavedSearch | SavedSearchSavedObject | null; + savedSearch: SavedSearch | null; /** App dependencies */ appDependencies: AiopsAppDependencies; } diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx index 8f4504070cc08..6c277456d2ec3 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx @@ -67,11 +67,11 @@ export const ExplainLogRateSpikesPage: FC = () => { ); const [globalState, setGlobalState] = useUrlState('_g'); - const [currentSavedSearch, setCurrentSavedSearch] = useState(savedSearch); + const [selectedSavedSearch, setSelectedSavedSearch] = useState(savedSearch); useEffect(() => { if (savedSearch) { - setCurrentSavedSearch(savedSearch); + setSelectedSavedSearch(savedSearch); } }, [savedSearch]); @@ -84,8 +84,8 @@ export const ExplainLogRateSpikesPage: FC = () => { }) => { // When the user loads a saved search and then clears or modifies the query // we should remove the saved search and replace it with the index pattern id - if (currentSavedSearch !== null) { - setCurrentSavedSearch(null); + if (selectedSavedSearch !== null) { + setSelectedSavedSearch(null); } setAiopsListState({ @@ -96,7 +96,7 @@ export const ExplainLogRateSpikesPage: FC = () => { filters: searchParams.filters, }); }, - [currentSavedSearch, aiopsListState, setAiopsListState] + [selectedSavedSearch, aiopsListState, setAiopsListState] ); const { @@ -108,7 +108,7 @@ export const ExplainLogRateSpikesPage: FC = () => { searchString, searchQuery, } = useData( - { currentDataView: dataView, currentSavedSearch }, + { selectedDataView: dataView, selectedSavedSearch }, aiopsListState, setGlobalState, currentSelectedChangePoint, diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx index 235aa1a518403..b554013393926 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx @@ -17,7 +17,6 @@ import { UI_SETTINGS } from '@kbn/data-plugin/common'; import { toMountPoint, wrapWithTheme } from '@kbn/kibana-react-plugin/public'; import { DataSourceContext } from '../../hooks/use_data_source'; -import { SavedSearchSavedObject } from '../../application/utils/search_utils'; import type { AiopsAppDependencies } from '../../hooks/use_aiops_app_context'; import { AIOPS_STORAGE_KEYS } from '../../types/storage'; import { AiopsAppContext } from '../../hooks/use_aiops_app_context'; @@ -28,7 +27,7 @@ const localStorage = new Storage(window.localStorage); export interface LogCategorizationAppStateProps { dataView: DataView; - savedSearch: SavedSearch | SavedSearchSavedObject | null; + savedSearch: SavedSearch | null; appDependencies: AiopsAppDependencies; } diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx index ab66a75754e63..feb6ef9c9b31c 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx @@ -52,13 +52,19 @@ export const LogCategorizationPage: FC = () => { const [selectedField, setSelectedField] = useState(); const [selectedCategory, setSelectedCategory] = useState(null); const [categories, setCategories] = useState(null); - const [currentSavedSearch, setCurrentSavedSearch] = useState(savedSearch); + const [selectedSavedSearch, setSelectedDataView] = useState(savedSearch); const [loading, setLoading] = useState(false); const [totalCount, setTotalCount] = useState(0); const [eventRate, setEventRate] = useState([]); const [pinnedCategory, setPinnedCategory] = useState(null); const [sparkLines, setSparkLines] = useState({}); + useEffect(() => { + if (savedSearch) { + setSelectedDataView(savedSearch); + } + }, [savedSearch]); + useEffect( function cancelRequestOnLeave() { return () => { @@ -77,8 +83,8 @@ export const LogCategorizationPage: FC = () => { }) => { // When the user loads saved search and then clear or modify the query // we should remove the saved search and replace it with the index pattern id - if (currentSavedSearch !== null) { - setCurrentSavedSearch(null); + if (selectedSavedSearch !== null) { + setSelectedDataView(null); } setAiopsListState({ @@ -89,7 +95,7 @@ export const LogCategorizationPage: FC = () => { filters: searchParams.filters, }); }, - [currentSavedSearch, aiopsListState, setAiopsListState] + [selectedSavedSearch, aiopsListState, setAiopsListState] ); const { @@ -102,7 +108,7 @@ export const LogCategorizationPage: FC = () => { searchQuery, intervalMs, } = useData( - { currentDataView: dataView, currentSavedSearch }, + { selectedDataView: dataView, selectedSavedSearch }, aiopsListState, setGlobalState, undefined, diff --git a/x-pack/plugins/aiops/public/hooks/use_data.ts b/x-pack/plugins/aiops/public/hooks/use_data.ts index 3cace69ed4e98..ff04b27b44e3c 100644 --- a/x-pack/plugins/aiops/public/hooks/use_data.ts +++ b/x-pack/plugins/aiops/public/hooks/use_data.ts @@ -16,10 +16,7 @@ import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker'; import type { DocumentStatsSearchStrategyParams } from '../get_document_stats'; import type { AiOpsIndexBasedAppState } from '../components/explain_log_rate_spikes/explain_log_rate_spikes_app_state'; -import { - getEsQueryFromSavedSearch, - SavedSearchSavedObject, -} from '../application/utils/search_utils'; +import { getEsQueryFromSavedSearch } from '../application/utils/search_utils'; import type { GroupTableItem } from '../components/spike_analysis_table/types'; import { useTimeBuckets } from './use_time_buckets'; @@ -31,9 +28,9 @@ const DEFAULT_BAR_TARGET = 75; export const useData = ( { - currentDataView, - currentSavedSearch, - }: { currentDataView: DataView; currentSavedSearch: SavedSearch | SavedSearchSavedObject | null }, + selectedDataView, + selectedSavedSearch, + }: { selectedDataView: DataView; selectedSavedSearch: SavedSearch | null }, aiopsListState: AiOpsIndexBasedAppState, onUpdate: (params: Dictionary) => void, selectedChangePoint?: ChangePoint, @@ -55,9 +52,9 @@ export const useData = ( /** Prepare required params to pass to search strategy **/ const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { const searchData = getEsQueryFromSavedSearch({ - dataView: currentDataView, + dataView: selectedDataView, uiSettings, - savedSearch: currentSavedSearch, + savedSearch: selectedSavedSearch, filterManager, }); @@ -82,8 +79,8 @@ export const useData = ( } // eslint-disable-next-line react-hooks/exhaustive-deps }, [ - currentSavedSearch?.id, - currentDataView.id, + selectedSavedSearch?.id, + selectedDataView.id, aiopsListState.searchString, aiopsListState.searchQueryLanguage, // eslint-disable-next-line react-hooks/exhaustive-deps @@ -96,7 +93,7 @@ export const useData = ( const _timeBuckets = useTimeBuckets(); const timefilter = useTimefilter({ - timeRangeSelector: currentDataView?.timeFieldName !== undefined, + timeRangeSelector: selectedDataView?.timeFieldName !== undefined, autoRefreshSelector: true, }); @@ -138,10 +135,10 @@ export const useData = ( earliest: timefilterActiveBounds.min?.valueOf(), latest: timefilterActiveBounds.max?.valueOf(), intervalMs: _timeBuckets.getInterval()?.asMilliseconds(), - index: currentDataView.getIndexPattern(), + index: selectedDataView.getIndexPattern(), searchQuery, - timeFieldName: currentDataView.timeFieldName, - runtimeFieldMap: currentDataView.getRuntimeMappings(), + timeFieldName: selectedDataView.timeFieldName, + runtimeFieldMap: selectedDataView.getRuntimeMappings(), }); setLastRefresh(Date.now()); } diff --git a/x-pack/plugins/aiops/public/hooks/use_data_source.ts b/x-pack/plugins/aiops/public/hooks/use_data_source.ts index e67aad07a338c..64cf4fe8ec521 100644 --- a/x-pack/plugins/aiops/public/hooks/use_data_source.ts +++ b/x-pack/plugins/aiops/public/hooks/use_data_source.ts @@ -7,12 +7,11 @@ import { createContext, useContext } from 'react'; import { DataView } from '@kbn/data-views-plugin/common'; -import { SavedSearch } from '@kbn/saved-search-plugin/public'; -import { SavedSearchSavedObject } from '../application/utils/search_utils'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; export const DataSourceContext = createContext<{ dataView: DataView | never; - savedSearch: SavedSearch | SavedSearchSavedObject | null; + savedSearch: SavedSearch | null; }>({ get dataView(): never { throw new Error('DataSourceContext is not implemented'); diff --git a/x-pack/plugins/data_visualizer/kibana.json b/x-pack/plugins/data_visualizer/kibana.json index 98d27ba9f0481..e7a136c1d5c83 100644 --- a/x-pack/plugins/data_visualizer/kibana.json +++ b/x-pack/plugins/data_visualizer/kibana.json @@ -32,7 +32,8 @@ "fieldFormats", "uiActions", "lens", - "cloudChat" + "cloudChat", + "savedSearch" ], "owner": { "name": "Machine Learning UI", diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts index fb782a02a08d4..5afa3869abd88 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/common/util/field_types_utils.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import { DataViewField } from '@kbn/data-views-plugin/public'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; -import { SupportedFieldType } from '../../../../common/types'; +import type { SupportedFieldType } from '../../../../common/types'; import { SUPPORTED_FIELD_TYPES } from '../../../../common/constants'; export const getJobTypeLabel = (type: string) => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 9f969fac23a38..ea72215a89598 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -36,6 +36,7 @@ import { } from '@kbn/ml-date-picker'; import { useStorage } from '@kbn/ml-local-storage'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import { useCurrentEuiTheme } from '../../../common/hooks/use_current_eui_theme'; import { DV_FROZEN_TIER_PREFERENCE, @@ -57,7 +58,7 @@ import { DataVisualizerIndexBasedPageUrlState, } from '../../types/index_data_visualizer_state'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; -import { SupportedFieldType, SavedSearchSavedObject } from '../../../../../common/types'; +import type { SupportedFieldType } from '../../../../../common/types'; import { useDataVisualizerKibana } from '../../../kibana_context'; import { FieldCountPanel } from '../../../common/components/field_count_panel'; import { DocumentCountContent } from '../../../common/components/document_count_content'; @@ -129,7 +130,7 @@ export const getDefaultDataVisualizerListState = ( export interface IndexDataVisualizerViewProps { currentDataView: DataView; - currentSavedSearch: SavedSearchSavedObject | null; + currentSavedSearch: SavedSearch | null; currentSessionId?: string; getAdditionalLinks?: GetAdditionalLinks; } @@ -178,12 +179,6 @@ export const IndexDataVisualizerView: FC = (dataVi const { currentDataView, currentSessionId, getAdditionalLinks } = dataVisualizerProps; - useEffect(() => { - if (dataVisualizerProps?.currentSavedSearch !== undefined) { - setCurrentSavedSearch(dataVisualizerProps?.currentSavedSearch); - } - }, [dataVisualizerProps?.currentSavedSearch]); - useEffect(() => { if (!currentDataView.isTimeBased()) { toasts.addWarning({ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index ac93003d1817e..9aeab958683e7 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -21,7 +21,7 @@ import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; import { isDefined } from '@kbn/ml-is-defined'; import { DataVisualizerFieldNamesFilter } from './field_name_filter'; import { DataVisualizerFieldTypeFilter } from './field_type_filter'; -import { SupportedFieldType } from '../../../../../common/types'; +import type { SupportedFieldType } from '../../../../../common/types'; import { SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import { createMergedEsQuery } from '../../utils/saved_search_utils'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx index 6534ace2343d8..d9cd0dfbaccfd 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/grid_embeddable/grid_embeddable.tsx @@ -27,8 +27,8 @@ import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-pl import type { Query } from '@kbn/es-query'; import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; import { DatePickerContextProvider } from '@kbn/ml-date-picker'; -import { SavedSearch } from '@kbn/discover-plugin/public'; -import { SamplingOption } from '../../../../../common/types/field_stats'; +import type { SavedSearch } from '@kbn/discover-plugin/public'; +import type { SamplingOption } from '../../../../../common/types/field_stats'; import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from './constants'; import { EmbeddableLoading } from './embeddable_loading_fallback'; import { DataVisualizerStartDependencies } from '../../../../plugin'; @@ -38,7 +38,7 @@ import { } from '../../../common/components/stats_table'; import { FieldVisConfig } from '../../../common/components/stats_table/types'; import { getDefaultDataVisualizerListState } from '../../components/index_data_visualizer_view/index_data_visualizer_view'; -import type { DataVisualizerTableState, SavedSearchSavedObject } from '../../../../../common/types'; +import type { DataVisualizerTableState } from '../../../../../common/types'; import type { DataVisualizerIndexBasedAppState } from '../../types/index_data_visualizer_state'; import { IndexBasedDataVisualizerExpandedRow } from '../../../common/components/expanded_row/index_based_expanded_row'; import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_data'; @@ -46,7 +46,7 @@ import { useDataVisualizerGridData } from '../../hooks/use_data_visualizer_grid_ export type DataVisualizerGridEmbeddableServices = [CoreStart, DataVisualizerStartDependencies]; export interface DataVisualizerGridInput { dataView: DataView; - savedSearch?: SavedSearch | SavedSearchSavedObject | null; + savedSearch?: SavedSearch | null; query?: Query; visibleFieldNames?: string[]; filters?: Filter[]; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index 858df7c6366b7..66902ed0fe190 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -98,7 +98,7 @@ export const useDataVisualizerGridData = ( ); /** Prepare required params to pass to search strategy **/ - const { searchQueryLanguage, searchString, searchQuery } = useMemo(() => { + const { searchQueryLanguage, searchString, searchQuery, queryOrAggregateQuery } = useMemo(() => { const filterManager = data.query.filterManager; const searchData = getEsQueryFromSavedSearch({ dataView: currentDataView, @@ -123,6 +123,7 @@ export const useDataVisualizerGridData = ( }; } else { return { + queryOrAggregateQuery: searchData.queryOrAggregateQuery, searchQuery: searchData.searchQuery, searchString: searchData.searchString, searchQueryLanguage: searchData.queryLanguage, @@ -563,6 +564,7 @@ export const useDataVisualizerGridData = ( progress: combinedProgress, overallStatsProgress, configs, + queryOrAggregateQuery, searchQueryLanguage, searchString, searchQuery, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx index f7c70625f0e65..7633d63286f8e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/index_data_visualizer.tsx @@ -11,7 +11,6 @@ import { useHistory, useLocation } from 'react-router-dom'; import { parse, stringify } from 'query-string'; import { isEqual } from 'lodash'; import { encode } from '@kbn/rison'; -import { SimpleSavedObject } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import { Storage } from '@kbn/kibana-utils-plugin/public'; import { @@ -33,14 +32,15 @@ import { type Dictionary, type SetUrlState, } from '@kbn/ml-url-state'; +import { getSavedSearch, type SavedSearch } from '@kbn/saved-search-plugin/public'; import { getCoreStart, getPluginsStart } from '../../kibana_services'; import { - IndexDataVisualizerViewProps, + type IndexDataVisualizerViewProps, IndexDataVisualizerView, } from './components/index_data_visualizer_view'; import { useDataVisualizerKibana } from '../kibana_context'; -import { GetAdditionalLinks } from '../common/components/results_links'; -import { DATA_VISUALIZER_APP_LOCATOR, IndexDataVisualizerLocatorParams } from './locator'; +import type { GetAdditionalLinks } from '../common/components/results_links'; +import { DATA_VISUALIZER_APP_LOCATOR, type IndexDataVisualizerLocatorParams } from './locator'; import { DATA_VISUALIZER_INDEX_VIEWER } from './constants/index_data_visualizer_viewer'; import { INDEX_DATA_VISUALIZER_NAME } from '../common/constants'; import { DV_STORAGE_KEYS } from './types/storage'; @@ -100,9 +100,7 @@ export const DataVisualizerStateContextProvider: FC(undefined); - const [currentSavedSearch, setCurrentSavedSearch] = useState | null>( - null - ); + const [currentSavedSearch, setCurrentSavedSearch] = useState(null); const [currentSessionId, setCurrentSessionId] = useState(undefined); @@ -161,21 +159,21 @@ export const DataVisualizerStateContextProvider: FC ref.type === 'index-pattern')?.id; - if (dataViewId !== undefined && savedSearch) { - try { - const dataView = await dataViews.get(dataViewId); - setCurrentSavedSearch(savedSearch); - setCurrentDataView(dataView); - } catch (e) { - toasts.addError(e, { - title: i18n.translate('xpack.dataVisualizer.index.dataViewErrorMessage', { - defaultMessage: 'Error finding data view', - }), - }); - } + const savedSearch = await getSavedSearch(savedSearchId, { + search, + savedObjectsClient, + }); + const dataView = savedSearch.searchSource.getField('index'); + + if (!dataView) { + toasts.addDanger({ + title: i18n.translate('xpack.dataVisualizer.index.dataViewErrorMessage', { + defaultMessage: 'Error finding data view', + }), + }); } + setCurrentSavedSearch(savedSearch); + setCurrentDataView(dataView); } catch (e) { toasts.addError(e, { title: i18n.translate('xpack.dataVisualizer.index.savedSearchErrorMessage', { @@ -192,7 +190,7 @@ export const DataVisualizerStateContextProvider: FC 49', language: 'lucene' } as Query, + filter: [ + { + meta: { + alias: null, + disabled: false, + negate: false, + params: [ + { + meta: { + alias: null, + disabled: false, + field: 'airline', + index: 'cb8808e0-9bfb-11ed-bb38-2b1bd55401e7', + key: 'airline', + negate: false, + params: { + query: 'ACA', + }, + type: 'phrase', + }, + query: { + match_phrase: { + airline: 'ACA', + }, + }, + }, + { + meta: { + alias: null, + disabled: false, + field: 'airline', + index: 'cb8808e0-9bfb-11ed-bb38-2b1bd55401e7', + key: 'airline', + negate: false, + params: { + query: 'FFT', + }, + type: 'phrase', + }, + query: { + match_phrase: { + airline: 'FFT', + }, + }, + }, + ] as FilterMetaParams, + // @ts-expect-error SavedSearch needs to be updated with CombinedFilterMeta + relation: 'OR', + type: 'combined', + index: 'cb8808e0-9bfb-11ed-bb38-2b1bd55401e7', + }, + query: {}, + $state: { + store: FilterStateStore.APP_STATE, + }, + }, + ], + }), +} as unknown as SavedSearch; + // @ts-expect-error We don't need the full object here const luceneSavedSearchObj: SavedSearchSavedObject = { attributes: { @@ -75,12 +145,27 @@ const kqlSavedSearch: SavedSearch = { title: 'farequote_filter_and_kuery', description: '', columns: ['_source'], - // @ts-expect-error this isn't a valid SavedSearch object... but does anyone care? - kibanaSavedObjectMeta: { - searchSourceJSON: - '{"highlightAll":true,"version":true,"query":{"query":"responsetime > 49","language":"kuery"},"filter":[{"meta":{"index":"90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0","negate":false,"disabled":false,"alias":null,"type":"phrase","key":"airline","value":"ASA","params":{"query":"ASA","type":"phrase"}},"query":{"match":{"airline":{"query":"ASA","type":"phrase"}}},"$state":{"store":"appState"}}],"indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index"}', - }, -}; + searchSource: createSearchSourceMock({ + index: mockDataView, + query: { query: 'responsetime > 49', language: 'kuery' } as Query, + filter: [ + { + meta: { + index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', + negate: false, + disabled: false, + alias: null, + type: 'phrase', + key: 'airline', + value: 'ASA', + params: { query: 'ASA', type: 'phrase' }, + }, + query: { match: { airline: { query: 'ASA', type: 'phrase' } } }, + $state: { store: FilterStateStore.APP_STATE }, + }, + ], + }), +} as unknown as SavedSearch; describe('getQueryFromSavedSearchObject()', () => { it('should return parsed searchSourceJSON with query and filter', () => { @@ -107,26 +192,24 @@ describe('getQueryFromSavedSearchObject()', () => { version: true, }); expect(getQueryFromSavedSearchObject(kqlSavedSearch)).toEqual({ + query: { query: 'responsetime > 49', language: 'kuery' }, + index: 'test-mock-data-view', filter: [ { - $state: { store: 'appState' }, meta: { - alias: null, - disabled: false, index: '90a978e0-1c80-11ec-b1d7-f7e5cf21b9e0', - key: 'airline', negate: false, - params: { query: 'ASA', type: 'phrase' }, + disabled: false, + alias: null, type: 'phrase', + key: 'airline', value: 'ASA', + params: { query: 'ASA', type: 'phrase' }, }, query: { match: { airline: { query: 'ASA', type: 'phrase' } } }, + $state: { store: 'appState' }, }, ], - highlightAll: true, - indexRefName: 'kibanaSavedObjectMeta.searchSourceJSON.index', - query: { language: 'kuery', query: 'responsetime > 49' }, - version: true, }); }); it('should return undefined if invalid searchSourceJSON', () => { @@ -222,27 +305,55 @@ describe('getEsQueryFromSavedSearch()', () => { expect( getEsQueryFromSavedSearch({ dataView: mockDataView, - savedSearch: luceneSavedSearchObj, + savedSearch: luceneSavedSearch, uiSettings: mockUiSettings, }) ).toEqual({ queryLanguage: 'lucene', + queryOrAggregateQuery: { + language: 'lucene', + query: 'responsetime > 49', + }, searchQuery: { bool: { - filter: [{ match_phrase: { airline: { query: 'ASA' } } }], - must: [{ query_string: { query: 'responsetime:>50' } }], + filter: [ + { + bool: { + should: [ + { + bool: { + must: [], + filter: [{ match_phrase: { airline: 'ACA' } }], + should: [], + must_not: [], + }, + }, + { + bool: { + must: [], + filter: [{ match_phrase: { airline: 'FFT' } }], + should: [], + must_not: [], + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + must: [{ query_string: { query: 'responsetime > 49' } }], must_not: [], should: [], }, }, - searchString: 'responsetime:>50', + searchString: 'responsetime > 49', }); }); it('should override original saved search with the provided query ', () => { expect( getEsQueryFromSavedSearch({ dataView: mockDataView, - savedSearch: luceneSavedSearchObj, + savedSearch: luceneSavedSearch, uiSettings: mockUiSettings, query: { query: 'responsetime:>100', @@ -251,9 +362,34 @@ describe('getEsQueryFromSavedSearch()', () => { }) ).toEqual({ queryLanguage: 'lucene', + queryOrAggregateQuery: { language: 'lucene', query: 'responsetime:>100' }, searchQuery: { bool: { - filter: [{ match_phrase: { airline: { query: 'ASA' } } }], + filter: [ + { + bool: { + should: [ + { + bool: { + must: [], + filter: [{ match_phrase: { airline: 'ACA' } }], + should: [], + must_not: [], + }, + }, + { + bool: { + must: [], + filter: [{ match_phrase: { airline: 'FFT' } }], + should: [], + must_not: [], + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], must: [{ query_string: { query: 'responsetime:>100' } }], must_not: [], should: [], @@ -267,7 +403,7 @@ describe('getEsQueryFromSavedSearch()', () => { expect( getEsQueryFromSavedSearch({ dataView: mockDataView, - savedSearch: luceneSavedSearchObj, + savedSearch: luceneSavedSearch, uiSettings: mockUiSettings, query: { query: 'responsetime:>100', @@ -299,6 +435,7 @@ describe('getEsQueryFromSavedSearch()', () => { }) ).toEqual({ queryLanguage: 'lucene', + queryOrAggregateQuery: { language: 'lucene', query: 'responsetime:>100' }, searchQuery: { bool: { filter: [], diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index a4678c5ec5dec..c3473c170b370 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -16,13 +16,13 @@ import { buildEsQuery, Query, Filter, + AggregateQuery, } from '@kbn/es-query'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { SearchSource } from '@kbn/data-plugin/common'; import { DataView } from '@kbn/data-views-plugin/public'; -import { SavedSearch } from '@kbn/discover-plugin/public'; -import { getEsQueryConfig } from '@kbn/data-plugin/common'; -import { FilterManager } from '@kbn/data-plugin/public'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; +import { getEsQueryConfig, isQuery, SearchSource } from '@kbn/data-plugin/common'; +import { FilterManager, mapAndFlattenFilters } from '@kbn/data-plugin/public'; import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; @@ -45,10 +45,12 @@ export function getDefaultQuery() { * from a saved search or saved search object */ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObject | SavedSearch) { - const search = isSavedSearchSavedObject(savedSearch) - ? savedSearch?.attributes?.kibanaSavedObjectMeta - : // @ts-ignore - savedSearch?.kibanaSavedObjectMeta; + if (!isSavedSearchSavedObject(savedSearch)) { + return savedSearch.searchSource.getSerializedFields(); + } + const search = + savedSearch?.attributes?.kibanaSavedObjectMeta ?? // @ts-ignore + savedSearch?.kibanaSavedObjectMeta; const parsed = typeof search?.searchSourceJSON === 'string' @@ -75,14 +77,14 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObjec * Should also form a valid query if only the query or filters is provided */ export function createMergedEsQuery( - query?: Query, + query?: Query | AggregateQuery | undefined, filters?: Filter[], dataView?: DataView, uiSettings?: IUiSettingsClient ) { let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); - if (query && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { + if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); if (query.query !== '') { combinedQuery = toElasticsearchQuery(ast, dataView); @@ -114,6 +116,14 @@ export function createMergedEsQuery( return combinedQuery; } +function getSavedSearchSource(savedSearch: SavedSearch) { + return savedSearch && + 'searchSource' in savedSearch && + savedSearch?.searchSource instanceof SearchSource + ? savedSearch.searchSource + : undefined; +} + /** * Extract query data from the saved search object * with overrides from the provided query data and/or filters @@ -128,7 +138,7 @@ export function getEsQueryFromSavedSearch({ }: { dataView: DataView; uiSettings: IUiSettingsClient; - savedSearch: SavedSearchSavedObject | SavedSearch | null | undefined; + savedSearch: SavedSearch | null | undefined; query?: Query; filters?: Filter[]; filterManager?: FilterManager; @@ -138,17 +148,13 @@ export function getEsQueryFromSavedSearch({ const userQuery = query; const userFilters = filters; + const savedSearchSource = getSavedSearchSource(savedSearch); + // If saved search has a search source with nested parent // e.g. a search coming from Dashboard saved search embeddable // which already combines both the saved search's original query/filters and the Dashboard's // then no need to process any further - if ( - savedSearch && - 'searchSource' in savedSearch && - savedSearch?.searchSource instanceof SearchSource && - savedSearch.searchSource.getParent() !== undefined && - userQuery - ) { + if (savedSearchSource && savedSearchSource.getParent() !== undefined && userQuery) { // Flattened query from search source may contain a clause that narrows the time range // which might interfere with global time pickers so we need to remove const savedQuery = @@ -168,13 +174,9 @@ export function getEsQueryFromSavedSearch({ }; } - // If saved search is an json object with the original query and filter - // retrieve the parsed query and filter - const savedSearchData = getQueryFromSavedSearchObject(savedSearch); - // If no saved search available, use user's query and filters - if (!savedSearchData && userQuery) { - if (filterManager && userFilters) filterManager.addFilters(userFilters, false); + if (!savedSearch && userQuery) { + if (filterManager && userFilters) filterManager.addFilters(userFilters); const combinedQuery = createMergedEsQuery( userQuery, @@ -190,13 +192,14 @@ export function getEsQueryFromSavedSearch({ }; } - // If saved search available, merge saved search with latest user query or filters + // If saved search available, merge saved search with the latest user query or filters // which might differ from extracted saved search data - if (savedSearchData) { + if (savedSearchSource) { const globalFilters = filterManager?.getGlobalFilters(); - const currentQuery = userQuery ?? savedSearchData?.query; - const currentFilters = userFilters ?? savedSearchData?.filter; - + // FIXME: Add support for AggregateQuery type #150091 + const currentQuery = userQuery ?? (savedSearchSource.getField('query') as Query); + const currentFilters = + userFilters ?? mapAndFlattenFilters(savedSearchSource.getField('filter') as Filter[]); if (filterManager) filterManager.setFilters(currentFilters); if (globalFilters) filterManager?.addFilters(globalFilters); @@ -209,8 +212,9 @@ export function getEsQueryFromSavedSearch({ return { searchQuery: combinedQuery, - searchString: currentQuery.query, - queryLanguage: currentQuery.language as SearchQueryLanguage, + searchString: currentQuery?.query ?? '', + queryLanguage: (currentQuery?.language as SearchQueryLanguage) ?? 'kuery', + queryOrAggregateQuery: currentQuery, }; } } diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json index fd61172e31587..7472c3910f7fe 100644 --- a/x-pack/plugins/data_visualizer/tsconfig.json +++ b/x-pack/plugins/data_visualizer/tsconfig.json @@ -56,6 +56,7 @@ "@kbn/ml-date-picker", "@kbn/ml-is-defined", "@kbn/ml-query-utils", + "@kbn/saved-search-plugin", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/ml/kibana.json b/x-pack/plugins/ml/kibana.json index b794f3ba39a03..bf734f14afe1d 100644 --- a/x-pack/plugins/ml/kibana.json +++ b/x-pack/plugins/ml/kibana.json @@ -50,6 +50,7 @@ "lens", "maps", "savedObjects", + "savedSearch", "usageCollection", "unifiedFieldList", "unifiedSearch" diff --git a/x-pack/plugins/ml/public/application/aiops/change_point_detection.tsx b/x-pack/plugins/ml/public/application/aiops/change_point_detection.tsx index 3ee53940add2b..b3d3a537b9be1 100644 --- a/x-pack/plugins/ml/public/application/aiops/change_point_detection.tsx +++ b/x-pack/plugins/ml/public/application/aiops/change_point_detection.tsx @@ -25,7 +25,7 @@ export const ChangePointDetectionPage: FC = () => { const context = useMlContext(); const dataView = context.currentDataView; - const savedSearch = context.currentSavedSearch; + const savedSearch = context.selectedSavedSearch; return ( <> diff --git a/x-pack/plugins/ml/public/application/aiops/explain_log_rate_spikes.tsx b/x-pack/plugins/ml/public/application/aiops/explain_log_rate_spikes.tsx index c7abf7385c3b0..031297c55929a 100644 --- a/x-pack/plugins/ml/public/application/aiops/explain_log_rate_spikes.tsx +++ b/x-pack/plugins/ml/public/application/aiops/explain_log_rate_spikes.tsx @@ -25,7 +25,7 @@ export const ExplainLogRateSpikesPage: FC = () => { const context = useMlContext(); const dataView = context.currentDataView; - const savedSearch = context.currentSavedSearch; + const savedSearch = context.selectedSavedSearch; return ( <> diff --git a/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx b/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx index a78a1e3815be8..cdc0c079d541c 100644 --- a/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx +++ b/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx @@ -25,7 +25,7 @@ export const LogCategorizationPage: FC = () => { const context = useMlContext(); const dataView = context.currentDataView; - const savedSearch = context.currentSavedSearch; + const savedSearch = context.selectedSavedSearch; return ( <> diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index d10eb2c8e63ec..5e5099937c7e1 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -78,6 +78,10 @@ const App: FC = ({ coreStart, deps, appMountParams }) => { config: coreStart.uiSettings!, setBreadcrumbs: coreStart.chrome!.setBreadcrumbs, redirectToMlAccessDeniedPage, + getSavedSearchDeps: { + search: deps.data.search, + savedObjectsClient: coreStart.savedObjects.client, + }, }; const services = { diff --git a/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts b/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts index a2ce2dcea2273..8cd013732b726 100644 --- a/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts +++ b/x-pack/plugins/ml/public/application/components/data_recognizer/data_recognizer.d.ts @@ -7,11 +7,11 @@ import { FC } from 'react'; import type { DataView } from '@kbn/data-views-plugin/public'; -import { SavedSearchSavedObject } from '../../../../common/types/kibana'; +import { type SavedSearch } from '@kbn/saved-search-plugin/public'; declare const DataRecognizer: FC<{ indexPattern: DataView; - savedSearch: SavedSearchSavedObject | null; + savedSearch: SavedSearch | null; results: { count: number; onChange?: Function; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/kibana_context_value.ts b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/kibana_context_value.ts index 642bc4baee712..1e783660a338e 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/kibana_context_value.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/kibana_context_value.ts @@ -8,7 +8,10 @@ import { dataViewMock } from './data_view'; import { dataViewsContractMock } from './data_view_contract'; import { kibanaConfigMock } from './kibana_config'; -import { savedSearchMock } from './saved_search'; +import { deprecatedSavedSearchSimpleObjMock } from './saved_search'; +import { SavedSearch } from '@kbn/saved-search-plugin/public'; + +const mockSavedSearch: SavedSearch = {} as unknown as SavedSearch; export const kibanaContextValueMock = { combinedQuery: { @@ -16,7 +19,8 @@ export const kibanaContextValueMock = { language: 'the-query-language', }, currentDataView: dataViewMock, - currentSavedSearch: savedSearchMock, + deprecatedSavedSearchObj: deprecatedSavedSearchSimpleObjMock, + selectedSavedSearch: mockSavedSearch, dataViewsContract: dataViewsContractMock, kibanaConfig: kibanaConfigMock, }; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/saved_search.ts b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/saved_search.ts index b9475996c79d8..f16b4f311f2e4 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/saved_search.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/__mocks__/saved_search.ts @@ -5,7 +5,7 @@ * 2.0. */ -export const savedSearchMock: any = { +export const deprecatedSavedSearchSimpleObjMock: any = { id: 'the-saved-search-id', type: 'search', attributes: { diff --git a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts index 8b755b02f99b9..65a14111a8a8f 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts @@ -7,13 +7,16 @@ import React from 'react'; import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import type { SavedSearchSavedObject } from '../../../../common/types/kibana'; import type { MlServicesContext } from '../../app'; export interface MlContextValue { combinedQuery: any; currentDataView: DataView; // TODO this should be DataView or null - currentSavedSearch: SavedSearchSavedObject | null; + // @deprecated currentSavedSearch is of SavedSearchSavedObject type, change to selectedSavedSearch + deprecatedSavedSearchObj: SavedSearchSavedObject | null; + selectedSavedSearch: SavedSearch | null; dataViewsContract: DataViewsContract; kibanaConfig: any; // IUiSettingsClient; kibanaVersion: string; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/use_current_saved_search.ts b/x-pack/plugins/ml/public/application/contexts/ml/use_current_saved_search.ts deleted file mode 100644 index 1d55d0d454725..0000000000000 --- a/x-pack/plugins/ml/public/application/contexts/ml/use_current_saved_search.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useContext } from 'react'; - -import { MlContext } from './ml_context'; - -export const useCurrentSavedSearch = () => { - const context = useContext(MlContext); - - if (context.currentSavedSearch === undefined) { - throw new Error('currentSavedSearch is undefined'); - } - - return context.currentSavedSearch; -}; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/use_ml_context.ts b/x-pack/plugins/ml/public/application/contexts/ml/use_ml_context.ts index 1a07cb0338855..f9ee39921087f 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/use_ml_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/use_ml_context.ts @@ -15,7 +15,10 @@ export const useMlContext = () => { if ( context.combinedQuery === undefined || context.currentDataView === undefined || - context.currentSavedSearch === undefined || + // @deprecated currentSavedSearch is of SavedSearchSavedObject type + // and should be migrated to selectedSavedSearch + context.deprecatedSavedSearchObj === undefined || + context.selectedSavedSearch === undefined || context.dataViewsContract === undefined || context.kibanaConfig === undefined ) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index 094cc405c7297..9da07dc0e6116 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -121,7 +121,7 @@ export const ConfigurationStepForm: FC = ({ setCurrentStep, }) => { const mlContext = useMlContext(); - const { currentSavedSearch, currentDataView } = mlContext; + const { currentDataView, selectedSavedSearch } = mlContext; const { savedSearchQuery, savedSearchQueryStr } = useSavedSearch(); const [fieldOptionsFetchFail, setFieldOptionsFetchFail] = useState(false); @@ -611,9 +611,9 @@ export const ConfigurationStepForm: FC = ({ )} - {savedSearchQuery !== null - ? currentSavedSearch?.attributes.title - : currentDataView.title} + {selectedSavedSearch !== null + ? selectedSavedSearch.title + : currentDataView.getName()} } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts index 37526b6e66ff8..ab7cd1d6c1c59 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts @@ -10,12 +10,13 @@ import { buildEsQuery, buildQueryFromFilters, decorateQuery, + Filter, fromKueryExpression, + Query, toElasticsearchQuery, } from '@kbn/es-query'; import { useMlContext } from '../../../../../contexts/ml'; import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search'; -import { getQueryFromSavedSearchObject } from '../../../../../util/index_utils'; // `undefined` is used for a non-initialized state // `null` is set if no saved search is used @@ -33,14 +34,16 @@ export function useSavedSearch() { const [savedSearchQueryStr, setSavedSearchQueryStr] = useState(undefined); const mlContext = useMlContext(); - const { currentSavedSearch, currentDataView, kibanaConfig } = mlContext; + const { currentDataView, kibanaConfig, selectedSavedSearch } = mlContext; const getQueryData = () => { let qry: any = {}; let qryString; - if (currentSavedSearch !== null) { - const { query, filter } = getQueryFromSavedSearchObject(currentSavedSearch); + if (selectedSavedSearch) { + // FIXME: Add support for AggregateQuery type #150091 + const query = selectedSavedSearch.searchSource.getField('query') as Query; + const filter = (selectedSavedSearch.searchSource.getField('filter') ?? []) as Filter[]; const queryLanguage = query.language; qryString = query.query; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx index 7d25031dc766c..ec08e25bb6f15 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/job_type/page.tsx @@ -21,7 +21,6 @@ import { ES_FIELD_TYPES } from '@kbn/field-types'; import { useMlKibana, useNavigateToPath } from '../../../../contexts/kibana'; import { useMlContext } from '../../../../contexts/ml'; -import { isSavedSearchSavedObject } from '../../../../../../common/types/kibana'; import { DataRecognizer } from '../../../../components/data_recognizer'; import { addItemToRecentlyAccessed } from '../../../../util/recently_accessed'; import { timeBasedIndexCheck } from '../../../../util/index_utils'; @@ -46,7 +45,7 @@ export const Page: FC = () => { const [recognizerResultsCount, setRecognizerResultsCount] = useState(0); - const { currentSavedSearch, currentDataView } = mlContext; + const { currentDataView, selectedSavedSearch } = mlContext; const isTimeBasedIndex = timeBasedIndexCheck(currentDataView); const hasGeoFields = useMemo( @@ -58,27 +57,27 @@ export const Page: FC = () => { [currentDataView] ); const indexWarningTitle = - !isTimeBasedIndex && isSavedSearchSavedObject(currentSavedSearch) + !isTimeBasedIndex && selectedSavedSearch ? i18n.translate( 'xpack.ml.newJob.wizard.jobType.dataViewFromSavedSearchNotTimeBasedMessage', { defaultMessage: '{savedSearchTitle} uses data view {dataViewName} which is not time based', values: { - savedSearchTitle: currentSavedSearch.attributes.title as string, - dataViewName: currentDataView.title, + savedSearchTitle: selectedSavedSearch.title ?? '', + dataViewName: currentDataView.getName(), }, } ) : i18n.translate('xpack.ml.newJob.wizard.jobType.dataViewNotTimeBasedMessage', { defaultMessage: 'Data view {dataViewName} is not time based', - values: { dataViewName: currentDataView.title }, + values: { dataViewName: currentDataView.getName() }, }); - const pageTitleLabel = isSavedSearchSavedObject(currentSavedSearch) + const pageTitleLabel = selectedSavedSearch ? i18n.translate('xpack.ml.newJob.wizard.jobType.savedSearchPageTitleLabel', { defaultMessage: 'saved search {savedSearchTitle}', - values: { savedSearchTitle: currentSavedSearch.attributes.title as string }, + values: { savedSearchTitle: selectedSavedSearch.title ?? '' }, }) : i18n.translate('xpack.ml.newJob.wizard.jobType.dataViewPageTitleLabel', { defaultMessage: 'data view {dataViewName}', @@ -93,23 +92,23 @@ export const Page: FC = () => { }; const getUrlParams = () => { - return !isSavedSearchSavedObject(currentSavedSearch) + return !selectedSavedSearch ? `?index=${currentDataView.id}` - : `?savedSearchId=${currentSavedSearch.id}`; + : `?savedSearchId=${selectedSavedSearch.id}`; }; const addSelectionToRecentlyAccessed = async () => { - const title = !isSavedSearchSavedObject(currentSavedSearch) - ? currentDataView.title - : (currentSavedSearch.attributes.title as string); + const title = !selectedSavedSearch + ? currentDataView.getName() + : selectedSavedSearch.title ?? ''; const mlLocator = share.url.locators.get(ML_APP_LOCATOR)!; const dataVisualizerLink = await mlLocator.getUrl( { page: ML_PAGES.DATA_VISUALIZER_INDEX_VIEWER, pageState: { - ...(currentSavedSearch?.id - ? { savedSearchId: currentSavedSearch.id } + ...(selectedSavedSearch?.id + ? { savedSearchId: selectedSavedSearch.id } : { index: currentDataView.id }), }, }, @@ -295,7 +294,7 @@ export const Page: FC = () => { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 4d27dba214242..57e0a6d75720e 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -64,7 +64,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { () => jobCreatorFactory(jobType)( mlContext.currentDataView, - mlContext.currentSavedSearch, + mlContext.deprecatedSavedSearchObj, mlContext.combinedQuery ), // eslint-disable-next-line react-hooks/exhaustive-deps @@ -148,7 +148,7 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { jobCreator.modelChangeAnnotations = true; } - if (mlContext.currentSavedSearch !== null) { + if (mlContext.selectedSavedSearch !== null) { // Jobs created from saved searches cannot be cloned in the wizard as the // ML job config holds no reference to the saved search ID. jobCreator.createdBy = null; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard_steps.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard_steps.tsx index 81e42a3e4062a..db8374f4bec44 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard_steps.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard_steps.tsx @@ -71,15 +71,15 @@ export const WizardSteps: FC = ({ currentStep, setCurrentStep }) => { const [advancedExpanded, setAdvancedExpanded] = useState(false); const [additionalExpanded, setAdditionalExpanded] = useState(false); function getSummaryStepTitle() { - if (mlContext.currentSavedSearch !== null) { + if (mlContext.selectedSavedSearch) { return i18n.translate('xpack.ml.newJob.wizard.stepComponentWrapper.summaryTitleSavedSearch', { defaultMessage: 'New job from saved search {title}', - values: { title: mlContext.currentSavedSearch.attributes.title as string }, + values: { title: mlContext.selectedSavedSearch.title ?? '' }, }); } else if (mlContext.currentDataView.id !== undefined) { return i18n.translate('xpack.ml.newJob.wizard.stepComponentWrapper.summaryTitleDataView', { defaultMessage: 'New job from data view {dataViewName}', - values: { dataViewName: mlContext.currentDataView.title }, + values: { dataViewName: mlContext.currentDataView.getName() }, }); } return ''; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx index 1dbf2b2e99a84..5881f76632e70 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx @@ -92,23 +92,18 @@ export const Page: FC = ({ moduleId, existingGroupIds }) => { const [jobsAwaitingNodeCount, setJobsAwaitingNodeCount] = useState(0); // #endregion - const { - currentSavedSearch: savedSearch, - currentDataView: dataView, - combinedQuery, - } = useMlContext(); - const pageTitle = - savedSearch !== null - ? i18n.translate('xpack.ml.newJob.recognize.savedSearchPageTitle', { - defaultMessage: 'saved search {savedSearchTitle}', - values: { savedSearchTitle: savedSearch.attributes.title as string }, - }) - : i18n.translate('xpack.ml.newJob.recognize.dataViewPageTitle', { - defaultMessage: 'data view {dataViewName}', - values: { dataViewName: dataView.getName() }, - }); - const displayQueryWarning = savedSearch !== null; - const tempQuery = savedSearch === null ? undefined : combinedQuery; + const { selectedSavedSearch, currentDataView: dataView, combinedQuery } = useMlContext(); + const pageTitle = selectedSavedSearch + ? i18n.translate('xpack.ml.newJob.recognize.savedSearchPageTitle', { + defaultMessage: 'saved search {savedSearchTitle}', + values: { savedSearchTitle: selectedSavedSearch.title ?? '' }, + }) + : i18n.translate('xpack.ml.newJob.recognize.dataViewPageTitle', { + defaultMessage: 'data view {dataViewName}', + values: { dataViewName: dataView.getName() }, + }); + const displayQueryWarning = selectedSavedSearch !== null; + const tempQuery = selectedSavedSearch === null ? undefined : combinedQuery; /** * Loads recognizer module configuration. diff --git a/x-pack/plugins/ml/public/application/routing/router.tsx b/x-pack/plugins/ml/public/application/routing/router.tsx index e050bea78a5c9..4b9dab76994ed 100644 --- a/x-pack/plugins/ml/public/application/routing/router.tsx +++ b/x-pack/plugins/ml/public/application/routing/router.tsx @@ -19,6 +19,8 @@ import type { DataViewsContract } from '@kbn/data-views-plugin/public'; import { EuiLoadingContent } from '@elastic/eui'; import { UrlStateProvider } from '@kbn/ml-url-state'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { SavedObjectsClientContract } from '@kbn/core/public'; import { MlNotificationsContextProvider } from '../contexts/ml/ml_notifications_context'; import { MlContext, MlContextValue } from '../contexts/ml'; @@ -65,6 +67,10 @@ export interface PageDependencies { dataViewsContract: DataViewsContract; setBreadcrumbs: ChromeStart['setBreadcrumbs']; redirectToMlAccessDeniedPage: () => Promise; + getSavedSearchDeps: { + search: DataPublicPluginStart['search']; + savedObjectsClient: SavedObjectsClientContract; + }; } export const PageLoader: FC<{ context: MlContextValue }> = ({ context, children }) => { diff --git a/x-pack/plugins/ml/public/application/routing/routes/access_denied.tsx b/x-pack/plugins/ml/public/application/routing/routes/access_denied.tsx index 1868c521c72fd..d07f760adf791 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/access_denied.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/access_denied.tsx @@ -30,7 +30,14 @@ export const accessDeniedRouteFactory = (): MlRoute => ({ }); const PageWrapper: FC = ({ deps }) => { - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, {}); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + {} + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/aiops/change_point_detection.tsx b/x-pack/plugins/ml/public/application/routing/routes/aiops/change_point_detection.tsx index 8d85c09f5ddf1..c825c579e6497 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/aiops/change_point_detection.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/aiops/change_point_detection.tsx @@ -43,10 +43,17 @@ export const changePointDetectionRouteFactory = ( const PageWrapper: FC = ({ location, deps }) => { const { index, savedSearchId }: Record = parse(location.search, { sort: false }); - const { context } = useResolver(index, savedSearchId, deps.config, deps.dataViewsContract, { - checkBasicLicense, - cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), - }); + const { context } = useResolver( + index, + savedSearchId, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/aiops/explain_log_rate_spikes.tsx b/x-pack/plugins/ml/public/application/routing/routes/aiops/explain_log_rate_spikes.tsx index ac9728f0c54a0..b946f1a651099 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/aiops/explain_log_rate_spikes.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/aiops/explain_log_rate_spikes.tsx @@ -50,15 +50,23 @@ export const explainLogRateSpikesRouteFactory = ( disabled: !AIOPS_ENABLED, }); -const PageWrapper: FC = ({ location, deps }) => { +const PageWrapper: FC = ({ location, deps, ...restProps }) => { const { redirectToMlAccessDeniedPage } = deps; const { index, savedSearchId }: Record = parse(location.search, { sort: false }); - const { context } = useResolver(index, savedSearchId, deps.config, deps.dataViewsContract, { - checkBasicLicense, - cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - }); + const { context } = useResolver( + index, + savedSearchId, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/aiops/log_categorization.tsx b/x-pack/plugins/ml/public/application/routing/routes/aiops/log_categorization.tsx index e3aebb87b5341..6002d86a69520 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/aiops/log_categorization.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/aiops/log_categorization.tsx @@ -50,11 +50,19 @@ const PageWrapper: FC = ({ location, deps }) => { const { redirectToMlAccessDeniedPage } = deps; const { index, savedSearchId }: Record = parse(location.search, { sort: false }); - const { context } = useResolver(index, savedSearchId, deps.config, deps.dataViewsContract, { - checkBasicLicense, - cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - }); + const { context } = useResolver( + index, + savedSearchId, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx index fa0899ea23475..9466c271f31b0 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx @@ -48,11 +48,18 @@ const PageWrapper: FC = ({ location, deps }) => { sort: false, }); - const { context } = useResolver(index, savedSearchId, deps.config, deps.dataViewsContract, { - ...basicResolvers(deps), - analyticsFields: () => - loadNewJobCapabilities(index, savedSearchId, deps.dataViewsContract, DATA_FRAME_ANALYTICS), - }); + const { context } = useResolver( + index, + savedSearchId, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + ...basicResolvers(deps), + analyticsFields: () => + loadNewJobCapabilities(index, savedSearchId, deps.dataViewsContract, DATA_FRAME_ANALYTICS), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_exploration.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_exploration.tsx index d47780553eede..1d4e483126e7f 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_exploration.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_exploration.tsx @@ -46,6 +46,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_jobs_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_jobs_list.tsx index fd3a365d01179..0b94eb490c778 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_jobs_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_jobs_list.tsx @@ -44,6 +44,7 @@ const PageWrapper: FC = ({ location, deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_map.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_map.tsx index faae919f722c8..853be56943341 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_map.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_map.tsx @@ -45,6 +45,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_source_selection.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_source_selection.tsx index e9ffcc509b153..6d255a8fb5fb3 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_source_selection.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_source_selection.tsx @@ -44,6 +44,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); diff --git a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/datavisualizer.tsx b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/datavisualizer.tsx index ad01c2d97b718..29f9063cddd38 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/datavisualizer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/datavisualizer.tsx @@ -35,11 +35,18 @@ export const selectorRouteFactory = ( const PageWrapper: FC = ({ location, deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkBasicLicense, - checkFindFileStructurePrivilege: () => - checkFindFileStructurePrivilegeResolver(redirectToMlAccessDeniedPage), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + checkFindFileStructurePrivilege: () => + checkFindFileStructurePrivilegeResolver(redirectToMlAccessDeniedPage), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/file_based.tsx b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/file_based.tsx index 3d048e8d40283..3e58a2b2bd08f 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/file_based.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/file_based.tsx @@ -45,12 +45,19 @@ export const fileBasedRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkBasicLicense, - cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), - checkFindFileStructurePrivilege: () => - checkFindFileStructurePrivilegeResolver(redirectToMlAccessDeniedPage), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), + checkFindFileStructurePrivilege: () => + checkFindFileStructurePrivilegeResolver(redirectToMlAccessDeniedPage), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx index 1ba865435280c..acd51501092b8 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/index_based.tsx @@ -47,11 +47,19 @@ const PageWrapper: FC = ({ location, deps }) => { const { redirectToMlAccessDeniedPage } = deps; const { index, savedSearchId }: Record = parse(location.search, { sort: false }); - const { context } = useResolver(index, savedSearchId, deps.config, deps.dataViewsContract, { - checkBasicLicense, - cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - }); + const { context } = useResolver( + index, + savedSearchId, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkBasicLicense, + cacheDataViewsContract: () => cacheDataViewsContract(deps.dataViewsContract), + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + } + ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index e7e672248f625..99c3f3ff38a68 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -75,6 +75,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, { ...basicResolvers(deps), jobs: mlJobService.loadJobsWrapper, diff --git a/x-pack/plugins/ml/public/application/routing/routes/jobs_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/jobs_list.tsx index 68c0cdbe6a24f..bcf8929c7f470 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/jobs_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/jobs_list.tsx @@ -50,6 +50,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); const timefilter = useTimefilter({ timeRangeSelector: false, autoRefreshSelector: true }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/from_lens.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/from_lens.tsx index a0690b4987bc0..0a9d1a30aad62 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/from_lens.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/from_lens.tsx @@ -30,9 +30,16 @@ const PageWrapper: FC = ({ location, deps }) => { } ); - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - redirect: () => resolver(lensId, vis, from, to, query, filters, layerIndex), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + redirect: () => resolver(lensId, vis, from, to, query, filters, layerIndex), + } + ); return ( {} diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/from_map.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/from_map.tsx index 3f1ebb04162e6..e63a00735b348 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/from_map.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/from_map.tsx @@ -36,10 +36,17 @@ const PageWrapper: FC = ({ location, deps }) => { sort: false, }); - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - redirect: () => - resolver(dashboard, dataViewId, embeddable, geoField, splitField, from, to, layer), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + redirect: () => + resolver(dashboard, dataViewId, embeddable, geoField, splitField, from, to, layer), + } + ); return ( {} diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 0383b48eabadc..fa35731e567c7 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -206,6 +206,7 @@ const PageWrapper: FC = ({ nextStepPath, deps, mode }) = undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, mode === MODE.NEW_JOB ? newJobResolvers : dataVizResolvers ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/job_type.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/job_type.tsx index e1d38875bc1dd..e4c9fa21cbd46 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/job_type.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/job_type.tsx @@ -41,6 +41,7 @@ const PageWrapper: FC = ({ location, deps }) => { savedSearchId, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/recognize.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/recognize.tsx index a68bb0d861a5c..1c1c54aebc4cc 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/recognize.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/recognize.tsx @@ -55,6 +55,8 @@ const PageWrapper: FC = ({ location, deps }) => { savedSearchId, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, + { ...basicResolvers(deps), existingJobsAndGroups: mlJobService.getJobAndGroupIds, @@ -77,7 +79,7 @@ const CheckViewOrCreateWrapper: FC = ({ location, deps }) => { const navigateToPath = useNavigateToPath(); // the single resolver checkViewOrCreateJobs redirects only. so will always reject - useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { + useResolver(undefined, undefined, deps.config, deps.dataViewsContract, deps.getSavedSearchDeps, { checkViewOrCreateJobs: () => checkViewOrCreateJobs(moduleId, dataViewId, createLinkWithUserDefaults, navigateToPath), }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx index 8a4640a346a85..4300d73ae010e 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx @@ -200,6 +200,7 @@ const PageWrapper: FC = ({ location, jobType, deps }) => { savedSearchId, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, { ...basicResolvers(deps), privileges: () => checkCreateJobsCapabilitiesResolver(redirectToJobsManagementPage), diff --git a/x-pack/plugins/ml/public/application/routing/routes/notifications.tsx b/x-pack/plugins/ml/public/application/routing/routes/notifications.tsx index cb7cbc0c48c6e..843cf54087c8f 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/notifications.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/notifications.tsx @@ -46,12 +46,20 @@ export const notificationsRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - getMlNodeCount, - loadMlServerInfo, - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + getMlNodeCount, + loadMlServerInfo, + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/overview.tsx b/x-pack/plugins/ml/public/application/routing/routes/overview.tsx index 139e4635ce412..111a1ae77916c 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/overview.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/overview.tsx @@ -50,12 +50,20 @@ export const overviewRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - getMlNodeCount, - loadMlServerInfo, - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + getMlNodeCount, + loadMlServerInfo, + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_list.tsx index e257f0cde5aac..2d43c5ab5675c 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_list.tsx @@ -41,11 +41,19 @@ export const calendarListRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - getMlNodeCount, - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + getMlNodeCount, + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_new_edit.tsx b/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_new_edit.tsx index 9753e790cd543..775ba25523a9d 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_new_edit.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/settings/calendar_new_edit.tsx @@ -85,11 +85,19 @@ const PageWrapper: FC = ({ location, mode, deps }) => { ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE ); - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - checkMlNodesAvailable: () => checkMlNodesAvailable(redirectToJobsManagementPage), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + checkMlNodesAvailable: () => checkMlNodesAvailable(redirectToJobsManagementPage), + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list.tsx index d4e8b61ef4870..abbe806716feb 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list.tsx @@ -41,11 +41,19 @@ export const filterListRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - getMlNodeCount, - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + getMlNodeCount, + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list_new_edit.tsx b/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list_new_edit.tsx index c501760080624..9afa6ec8c8557 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list_new_edit.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/settings/filter_list_new_edit.tsx @@ -89,11 +89,19 @@ const PageWrapper: FC = ({ location, mode, deps }) => { ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE ); - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - checkMlNodesAvailable: () => checkMlNodesAvailable(redirectToJobsManagementPage), - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + checkMlNodesAvailable: () => checkMlNodesAvailable(redirectToJobsManagementPage), + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/settings/settings.tsx b/x-pack/plugins/ml/public/application/routing/routes/settings/settings.tsx index 2e994cc4f85f9..2b9af849e2157 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/settings/settings.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/settings/settings.tsx @@ -41,11 +41,19 @@ export const settingsRouteFactory = ( const PageWrapper: FC = ({ deps }) => { const { redirectToMlAccessDeniedPage } = deps; - const { context } = useResolver(undefined, undefined, deps.config, deps.dataViewsContract, { - checkFullLicense, - checkGetJobsCapabilities: () => checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), - getMlNodeCount, - }); + const { context } = useResolver( + undefined, + undefined, + deps.config, + deps.dataViewsContract, + deps.getSavedSearchDeps, + { + checkFullLicense, + checkGetJobsCapabilities: () => + checkGetJobsCapabilitiesResolver(redirectToMlAccessDeniedPage), + getMlNodeCount, + } + ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: false }); diff --git a/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.tsx index e89a2cff13873..729bb5ae821d8 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.tsx @@ -79,6 +79,7 @@ const PageWrapper: FC = ({ deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, { ...basicResolvers(deps), jobs: mlJobService.loadJobsWrapper, diff --git a/x-pack/plugins/ml/public/application/routing/routes/trained_models/models_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/trained_models/models_list.tsx index e2b0b2709df52..46fd368829344 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/trained_models/models_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/trained_models/models_list.tsx @@ -47,6 +47,7 @@ const PageWrapper: FC = ({ location, deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/trained_models/nodes_list.tsx b/x-pack/plugins/ml/public/application/routing/routes/trained_models/nodes_list.tsx index e6003f161b1c6..78d99daf0d2d7 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/trained_models/nodes_list.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/trained_models/nodes_list.tsx @@ -46,6 +46,7 @@ const PageWrapper: FC = ({ location, deps }) => { undefined, deps.config, deps.dataViewsContract, + deps.getSavedSearchDeps, basicResolvers(deps) ); useTimefilter({ timeRangeSelector: false, autoRefreshSelector: true }); diff --git a/x-pack/plugins/ml/public/application/routing/use_resolver.test.ts b/x-pack/plugins/ml/public/application/routing/use_resolver.test.ts index 714df97142cca..a75448459be96 100644 --- a/x-pack/plugins/ml/public/application/routing/use_resolver.test.ts +++ b/x-pack/plugins/ml/public/application/routing/use_resolver.test.ts @@ -13,7 +13,7 @@ import { useCreateAndNavigateToMlLink } from '../contexts/kibana/use_create_url' import { useNotifications } from '../contexts/kibana'; import type { DataViewsContract } from '@kbn/data-views-plugin/public'; -import { useResolver } from './use_resolver'; +import { type GetSavedSearchPageDeps, useResolver } from './use_resolver'; jest.mock('../contexts/kibana/use_create_url', () => { return { @@ -47,7 +47,14 @@ describe('useResolver', () => { it('should accept undefined as dataViewId and savedSearchId.', async () => { const { result, waitForNextUpdate } = renderHook(() => - useResolver(undefined, undefined, {} as IUiSettingsClient, {} as DataViewsContract, {}) + useResolver( + undefined, + undefined, + {} as IUiSettingsClient, + {} as DataViewsContract, + {} as GetSavedSearchPageDeps, + {} + ) ); await act(async () => { @@ -66,9 +73,10 @@ describe('useResolver', () => { }, }, currentDataView: null, - currentSavedSearch: null, + deprecatedSavedSearchObj: null, dataViewsContract: {}, kibanaConfig: {}, + selectedSavedSearch: null, }, results: {}, }); @@ -78,7 +86,14 @@ describe('useResolver', () => { it('should add an error toast and redirect if dataViewId is an empty string.', async () => { const { result } = renderHook(() => - useResolver('', undefined, {} as IUiSettingsClient, {} as DataViewsContract, {}) + useResolver( + '', + undefined, + {} as IUiSettingsClient, + {} as DataViewsContract, + {} as GetSavedSearchPageDeps, + {} + ) ); await act(async () => {}); diff --git a/x-pack/plugins/ml/public/application/routing/use_resolver.ts b/x-pack/plugins/ml/public/application/routing/use_resolver.ts index 6e16b1d42e6f6..0f999329b2fa1 100644 --- a/x-pack/plugins/ml/public/application/routing/use_resolver.ts +++ b/x-pack/plugins/ml/public/application/routing/use_resolver.ts @@ -6,21 +6,27 @@ */ import { useEffect, useState } from 'react'; -import { IUiSettingsClient } from '@kbn/core/public'; +import type { IUiSettingsClient, SavedObjectsClientContract } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import type { DataViewsContract } from '@kbn/data-views-plugin/public'; +import { getSavedSearch } from '@kbn/saved-search-plugin/public'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import { getDataViewById, getDataViewAndSavedSearch, - DataViewAndSavedSearch, + type DataViewAndSavedSearch, } from '../util/index_utils'; import { createSearchItems } from '../jobs/new_job/utils/new_job_utils'; -import { ResolverResults, Resolvers } from './resolvers'; -import { MlContextValue } from '../contexts/ml'; +import type { ResolverResults, Resolvers } from './resolvers'; +import type { MlContextValue } from '../contexts/ml'; import { useNotifications } from '../contexts/kibana'; import { useCreateAndNavigateToMlLink } from '../contexts/kibana/use_create_url'; import { ML_PAGES } from '../../../common/constants/locator'; +export interface GetSavedSearchPageDeps { + search: DataPublicPluginStart['search']; + savedObjectsClient: SavedObjectsClientContract; +} /** * Hook to resolve route specific requirements * @param dataViewId optional Kibana data view id, used for wizards @@ -34,6 +40,10 @@ export const useResolver = ( savedSearchId: string | undefined, config: IUiSettingsClient, dataViewsContract: DataViewsContract, + getSavedSearchDeps: { + search: DataPublicPluginStart['search']; + savedObjectsClient: SavedObjectsClientContract; + }, resolvers: Resolvers ): { context: MlContextValue; results: ResolverResults } => { const notifications = useNotifications(); @@ -76,25 +86,28 @@ export const useResolver = ( savedSearch: null, dataView: null, }; + let savedSearch = null; if (savedSearchId !== undefined) { + savedSearch = await getSavedSearch(savedSearchId, getSavedSearchDeps); dataViewAndSavedSearch = await getDataViewAndSavedSearch(savedSearchId); } else if (dataViewId !== undefined) { dataViewAndSavedSearch.dataView = await getDataViewById(dataViewId); } - const { savedSearch, dataView } = dataViewAndSavedSearch; + const { savedSearch: deprecatedSavedSearchObj, dataView } = dataViewAndSavedSearch; const { combinedQuery } = createSearchItems( config, dataView !== null ? dataView : undefined, - savedSearch + deprecatedSavedSearchObj ); setContext({ combinedQuery, currentDataView: dataView, - currentSavedSearch: savedSearch, + deprecatedSavedSearchObj, + selectedSavedSearch: savedSearch, dataViewsContract, kibanaConfig: config, }); diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index d613793099257..5dc4bfd67dc58 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -79,5 +79,6 @@ "@kbn/shared-ux-link-redirect-app", "@kbn/react-field", "@kbn/unified-field-list-plugin", + "@kbn/saved-search-plugin", ], } From 8eb89aaf19dd9f48912157b33d2e39bfb14ed1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Mon, 6 Feb 2023 19:21:08 +0100 Subject: [PATCH 005/134] Add FTRs to the `cloud_links` plugin (#150220) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Resolves https://github.com/elastic/kibana/issues/149206 --- .buildkite/ftr_configs.yml | 1 + x-pack/test/functional_cloud/config.ts | 63 ++++++++ .../functional_cloud/ftr_provider_context.ts | 13 ++ x-pack/test/functional_cloud/services.ts | 10 ++ x-pack/test/functional_cloud/test_utils.ts | 138 ++++++++++++++++++ .../functional_cloud/tests/cloud_links.ts | 62 ++++++++ x-pack/test/functional_cloud/tests/index.ts | 14 ++ 7 files changed, 301 insertions(+) create mode 100644 x-pack/test/functional_cloud/config.ts create mode 100644 x-pack/test/functional_cloud/ftr_provider_context.ts create mode 100644 x-pack/test/functional_cloud/services.ts create mode 100644 x-pack/test/functional_cloud/test_utils.ts create mode 100644 x-pack/test/functional_cloud/tests/cloud_links.ts create mode 100644 x-pack/test/functional_cloud/tests/index.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 0b56007cf9aeb..5c3f6aef0271e 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -243,6 +243,7 @@ enabled: - x-pack/test/functional/config_security_basic.ts - x-pack/test/functional/config.ccs.ts - x-pack/test/functional/config.firefox.js + - x-pack/test/functional_cloud/config.ts - x-pack/test/kubernetes_security/basic/config.ts - x-pack/test/licensing_plugin/config.public.ts - x-pack/test/licensing_plugin/config.ts diff --git a/x-pack/test/functional_cloud/config.ts b/x-pack/test/functional_cloud/config.ts new file mode 100644 index 0000000000000..db0f19f9c21cc --- /dev/null +++ b/x-pack/test/functional_cloud/config.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { resolve } from 'path'; +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const functionalConfig = await readConfigFile(require.resolve('../functional/config.base.js')); + + const kibanaPort = functionalConfig.get('servers.kibana.port'); + const idpPath = resolve( + __dirname, + '../security_api_integration/fixtures/saml/saml_provider/metadata.xml' + ); + const samlIdPPlugin = resolve( + __dirname, + '../security_api_integration/fixtures/saml/saml_provider' + ); + + return { + ...functionalConfig.getAll(), + rootTags: ['skipCloud'], + testFiles: [require.resolve('./tests')], + security: { disableTestUser: true }, + junit: { + reportName: 'Cloud Integrations Functional Tests', + }, + esTestCluster: { + ...functionalConfig.get('esTestCluster'), + serverArgs: [ + ...functionalConfig.get('esTestCluster.serverArgs'), + 'xpack.security.authc.token.enabled=true', + 'xpack.security.authc.realms.saml.cloud-saml-kibana.order=0', + `xpack.security.authc.realms.saml.cloud-saml-kibana.idp.metadata.path=${idpPath}`, + 'xpack.security.authc.realms.saml.cloud-saml-kibana.idp.entity_id=http://www.elastic.co/saml1', + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.entity_id=http://localhost:${kibanaPort}`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.logout=http://localhost:${kibanaPort}/logout`, + `xpack.security.authc.realms.saml.cloud-saml-kibana.sp.acs=http://localhost:${kibanaPort}/api/security/saml/callback`, + 'xpack.security.authc.realms.saml.cloud-saml-kibana.attributes.principal=urn:oid:0.0.7', + ], + }, + kbnTestServer: { + ...functionalConfig.get('kbnTestServer'), + serverArgs: [ + ...functionalConfig.get('kbnTestServer.serverArgs'), + `--plugin-path=${samlIdPPlugin}`, + '--xpack.cloud.id=ftr_fake_cloud_id', + '--xpack.cloud.base_url=https://cloud.elastic.co', + '--xpack.cloud.deployment_url=/deployments/deploymentId', + '--xpack.cloud.organization_url=/organization/organizationId', + '--xpack.cloud.profile_url=/user/userId', + '--xpack.security.authc.selector.enabled=false', + `--xpack.security.authc.providers=${JSON.stringify({ + basic: { 'cloud-basic': { order: 1 } }, + saml: { 'cloud-saml-kibana': { order: 0, realm: 'cloud-saml-kibana' } }, + })}`, + ], + }, + }; +} diff --git a/x-pack/test/functional_cloud/ftr_provider_context.ts b/x-pack/test/functional_cloud/ftr_provider_context.ts new file mode 100644 index 0000000000000..d6c0afa5ceffd --- /dev/null +++ b/x-pack/test/functional_cloud/ftr_provider_context.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GenericFtrProviderContext } from '@kbn/test'; +import { pageObjects } from '../functional/page_objects'; +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; +export { pageObjects }; diff --git a/x-pack/test/functional_cloud/services.ts b/x-pack/test/functional_cloud/services.ts new file mode 100644 index 0000000000000..9508ce5eba16d --- /dev/null +++ b/x-pack/test/functional_cloud/services.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { services as functionalServices } from '../functional/services'; + +export const services = functionalServices; diff --git a/x-pack/test/functional_cloud/test_utils.ts b/x-pack/test/functional_cloud/test_utils.ts new file mode 100644 index 0000000000000..65d0d7ece02c4 --- /dev/null +++ b/x-pack/test/functional_cloud/test_utils.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Fs from 'fs/promises'; +import Path from 'path'; +import { isEqualWith } from 'lodash'; +import type { Ecs, KibanaExecutionContext } from '@kbn/core/server'; +import type { RetryService } from '@kbn/ftr-common-functional-services'; +import { concatMap, defer, filter, firstValueFrom, ReplaySubject, scan, timeout } from 'rxjs'; + +export const logFilePath = Path.resolve(__dirname, './kibana.log'); +export const ANY = Symbol('any'); + +let logstream$: ReplaySubject | undefined; + +export function getExecutionContextFromLogRecord(record: Ecs | undefined): KibanaExecutionContext { + if (record?.log?.logger !== 'execution_context' || !record?.message) { + throw new Error(`The record is not an entry of execution context`); + } + return JSON.parse(record.message); +} + +export function isExecutionContextLog( + record: Ecs | undefined, + executionContext: KibanaExecutionContext +) { + try { + const object = getExecutionContextFromLogRecord(record); + return isEqualWith(object, executionContext, function customizer(obj1: any, obj2: any) { + if (obj2 === ANY) return true; + }); + } catch (e) { + return false; + } +} + +// to avoid splitting log record containing \n symbol +const endOfLine = /(?<=})\s*\n/; +export async function assertLogContains({ + description, + predicate, + retry, +}: { + description: string; + predicate: (record: Ecs) => boolean; + retry: RetryService; +}): Promise { + // logs are written to disk asynchronously. I sacrificed performance to reduce flakiness. + await retry.waitFor(description, async () => { + if (!logstream$) { + logstream$ = getLogstream$(); + } + try { + await firstValueFrom(logstream$.pipe(filter(predicate), timeout(5_000))); + return true; + } catch (err) { + return false; + } + }); +} + +/** + * Creates an observable that continuously tails the log file. + */ +function getLogstream$(): ReplaySubject { + const stream$ = new ReplaySubject(); + + defer(async function* () { + const fd = await Fs.open(logFilePath, 'rs'); + while (!stream$.isStopped) { + const { bytesRead, buffer } = await fd.read(); + if (bytesRead) { + yield buffer.toString('utf8', 0, bytesRead); + } + } + await fd.close(); + }) + .pipe( + scan( + ({ buffer }, chunk) => { + const logString = buffer.concat(chunk); + const lines = logString.split(endOfLine); + const lastLine = lines.pop(); + const records = lines.map((s) => JSON.parse(s)); + + let leftover = ''; + if (lastLine) { + try { + const validRecord = JSON.parse(lastLine); + records.push(validRecord); + } catch (err) { + leftover = lastLine; + } + } + + return { buffer: leftover, records }; + }, + { + records: [], // The ECS entries in the logs + buffer: '', // Accumulated leftovers from the previous operation + } + ), + concatMap(({ records }) => records) + ) + .subscribe(stream$); + + // let the content start flowing + stream$.subscribe(); + + return stream$; +} + +export function closeLogstream() { + logstream$?.complete(); + logstream$ = undefined; +} + +/** + * Truncates the log file to avoid tests looking at the logs from previous executions. + */ +export async function clearLogFile() { + closeLogstream(); + await Fs.writeFile(logFilePath, '', 'utf8'); + await forceSyncLogFile(); + logstream$ = getLogstream$(); +} + +/** + * Force the completion of all the pending I/O operations in the OS related to the log file. + */ +export async function forceSyncLogFile() { + const fileDescriptor = await Fs.open(logFilePath); + await fileDescriptor.datasync(); + await fileDescriptor.close(); +} diff --git a/x-pack/test/functional_cloud/tests/cloud_links.ts b/x-pack/test/functional_cloud/tests/cloud_links.ts new file mode 100644 index 0000000000000..4eabec139cbe1 --- /dev/null +++ b/x-pack/test/functional_cloud/tests/cloud_links.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const find = getService('find'); + const PageObjects = getPageObjects(['common', 'header']); + + describe('Cloud Links integration', function () { + before(async () => { + // Create role mapping so user gets superuser access + await getService('esSupertest') + .post('/_security/role_mapping/cloud-saml-kibana') + .send({ + roles: ['superuser'], + enabled: true, + rules: { field: { 'realm.name': 'cloud-saml-kibana' } }, + }) + .expect(200); + }); + + beforeEach(async () => { + await PageObjects.common.navigateToUrl('home'); + await PageObjects.header.waitUntilLoadingHasFinished(); + }); + + it('The button "Setup guides" is loaded', async () => { + expect(await find.byCssSelector('[data-test-subj="guideButtonRedirect"]')).to.not.be(null); + const cloudLink = await find.byLinkText('Setup guides'); + expect(cloudLink).to.not.be(null); + }); + + it('"Manage this deployment" is appended to the nav list', async () => { + await PageObjects.common.clickAndValidate('toggleNavButton', 'collapsibleNavCustomNavLink'); + const cloudLink = await find.byLinkText('Manage this deployment'); + expect(cloudLink).to.not.be(null); + }); + + describe('Fills up the user menu items', () => { + it('Shows the button Edit profile', async () => { + await PageObjects.common.clickAndValidate('userMenuButton', 'userMenuLink__Edit profile'); + const cloudLink = await find.byLinkText('Edit profile'); + expect(cloudLink).to.not.be(null); + }); + + it('Shows the button Account & Billing', async () => { + await PageObjects.common.clickAndValidate( + 'userMenuButton', + 'userMenuLink__Account & Billing' + ); + const cloudLink = await find.byLinkText('Account & Billing'); + expect(cloudLink).to.not.be(null); + }); + }); + }); +} diff --git a/x-pack/test/functional_cloud/tests/index.ts b/x-pack/test/functional_cloud/tests/index.ts new file mode 100644 index 0000000000000..c2609c5c2cac3 --- /dev/null +++ b/x-pack/test/functional_cloud/tests/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('Cloud Integrations', function () { + loadTestFile(require.resolve('./cloud_links')); + }); +} From 85b481bd38eeb4c2a0a3e6784d601d72774edc15 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 6 Feb 2023 13:23:21 -0500 Subject: [PATCH 006/134] Osquery: Update exported fields reference for osquery 5.7.0 (#150216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Update exported fields reference for osquery 5.7.0. ## Related PR - Requires https://github.com/elastic/beats/pull/34468 - Requires https://github.com/elastic/integrations/pull/5175 Co-authored-by: Patryk Kopyciński --- .../exported-fields-reference.asciidoc | 142 ++++++++++++++---- 1 file changed, 113 insertions(+), 29 deletions(-) diff --git a/docs/osquery/exported-fields-reference.asciidoc b/docs/osquery/exported-fields-reference.asciidoc index c27b6e67a4062..fc16ec3e0d9d0 100644 --- a/docs/osquery/exported-fields-reference.asciidoc +++ b/docs/osquery/exported-fields-reference.asciidoc @@ -82,7 +82,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *activity* - keyword, number.long -* _unified_log.activity_ - the activity ID associate with the entry. +* _unified_log.activity_ - the activity ID associate with the entry *actual* - keyword, number.long @@ -101,7 +101,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _arp_cache.address_ - IPv4 address target * _dns_resolvers.address_ - Resolver IP/IPv6 address * _etc_hosts.address_ - IP address mapping -* _fbsd_kmods.address_ - Kernel module address * _interface_addresses.address_ - Specific address for interface * _kernel_modules.address_ - Kernel module address * _listening_ports.address_ - Specific address for bind @@ -187,7 +186,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _deb_packages.arch_ - Package architecture * _docker_version.arch_ - Hardware architecture * _os_version.arch_ - OS Architecture -* _pkg_packages.arch_ - Architecture(s) supported * _rpm_packages.arch_ - Architecture(s) supported * _seccomp_events.arch_ - Information about the CPU architecture * _signature.arch_ - If applicable, the arch of the signed code @@ -247,6 +245,42 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _chassis_info.audible_alarm_ - If TRUE, the frame is equipped with an audible alarm. +*audit_account_logon* - keyword, number.long + +* _security_profile_info.audit_account_logon_ - Determines whether the operating system MUST audit each time this computer validates the credentials of an account + +*audit_account_manage* - keyword, number.long + +* _security_profile_info.audit_account_manage_ - Determines whether the operating system MUST audit each event of account management on a computer + +*audit_ds_access* - keyword, number.long + +* _security_profile_info.audit_ds_access_ - Determines whether the operating system MUST audit each instance of user attempts to access an Active Directory object that has its own system access control list (SACL) specified + +*audit_logon_events* - keyword, number.long + +* _security_profile_info.audit_logon_events_ - Determines whether the operating system MUST audit each instance of a user attempt to log on or log off this computer + +*audit_object_access* - keyword, number.long + +* _security_profile_info.audit_object_access_ - Determines whether the operating system MUST audit each instance of user attempts to access a non-Active Directory object that has its own SACL specified + +*audit_policy_change* - keyword, number.long + +* _security_profile_info.audit_policy_change_ - Determines whether the operating system MUST audit each instance of user attempts to change user rights assignment policy, audit policy, account policy, or trust policy + +*audit_privilege_use* - keyword, number.long + +* _security_profile_info.audit_privilege_use_ - Determines whether the operating system MUST audit each instance of user attempts to exercise a user right + +*audit_process_tracking* - keyword, number.long + +* _security_profile_info.audit_process_tracking_ - Determines whether the operating system MUST audit process-related events + +*audit_system_events* - keyword, number.long + +* _security_profile_info.audit_system_events_ - Determines whether the operating system MUST audit System Change, System Startup, System Shutdown, Authentication Component Load, and Loss or Excess of Security events + *auid* - keyword * _process_events.auid_ - Audit User ID at process start @@ -625,7 +659,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _ntfs_journal_events.category_ - The category that the event originated from * _power_sensors.category_ - The sensor category: currents, voltage, wattage * _system_extensions.category_ - System extension category -* _unified_log.category_ - The category of the os_log_t used +* _unified_log.category_ - the category of the os_log_t used * _yara_events.category_ - The category of the file *cdhash* - keyword, text.text @@ -731,6 +765,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _wmi_filter_consumer_binding.class_ - The name of the class. * _wmi_script_event_consumers.class_ - The name of the class. +*clear_text_password* - keyword, number.long + +* _security_profile_info.clear_text_password_ - Determines whether passwords MUST be stored by using reversible encryption + *client_app_id* - keyword, text.text * _windows_update_history.client_app_id_ - Identifier of the client application that processed an update @@ -767,6 +805,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _os_version.codename_ - OS version codename +*codesigning_flags* - keyword, text.text + +* _es_process_events.codesigning_flags_ - Codesigning flags matching one of these options, in a comma separated list: NOT_VALID, ADHOC, NOT_RUNTIME, INSTALLER. See kern/cs_blobs.h in XNU for descriptions. + *collect_cross_processes* - keyword, number.long * _carbon_black_info.collect_cross_processes_ - If the sensor is configured to cross process events @@ -848,7 +890,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _authorized_keys.comment_ - Optional comment * _docker_image_history.comment_ - Instruction comment * _etc_protocols.comment_ - Comment with protocol description -* _etc_services.comment_ - Optional comment for a service +* _etc_services.comment_ - Optional comment for a service. * _groups.comment_ - Remarks or comments associated with the group * _keychain_items.comment_ - Optional keychain comment @@ -1092,7 +1134,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _docker_image_history.created_ - Time of creation as UNIX time * _docker_images.created_ - Time of creation as UNIX time * _docker_networks.created_ - Time of creation as UNIX time -* _keychain_items.created_ - Data item was created +* _keychain_items.created_ - Date item was created *created_at* - keyword, text.text @@ -1590,6 +1632,14 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _processes.elevated_token_ - Process uses elevated token yes=1, no=0 +*enable_admin_account* - keyword, number.long + +* _security_profile_info.enable_admin_account_ - Determines whether the Administrator account on the local computer is enabled + +*enable_guest_account* - keyword, number.long + +* _security_profile_info.enable_guest_account_ - Determines whether the Guest account on the local computer is enabled + *enable_ipv6* - keyword, number.long * _docker_networks.enable_ipv6_ - 1 if IPv6 is enabled on this network. 0 otherwise @@ -1949,7 +1999,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *firmware_type* - keyword, text.text -* _platform_info.firmware_type_ - The type of firmware (Uefi, Bios, Unknown). +* _platform_info.firmware_type_ - The type of firmware (uefi, bios, iboot, openfirmware, unknown). *firmware_version* - keyword, text.text @@ -1972,10 +2022,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _pipes.flags_ - The flags indicating whether this pipe connection is a server or client end, and if the pipe for sending messages or bytes * _routes.flags_ - Flags to describe route -*flatsize* - keyword, number.long - -* _pkg_packages.flatsize_ - Package size in bytes - *folder_id* - keyword, text.text * _ycloud_instance_metadata.folder_id_ - Folder identifier for the VM @@ -1984,6 +2030,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _systemd_units.following_ - The name of another unit that this unit follows in state +*force_logoff_when_expire* - keyword, number.long + +* _security_profile_info.force_logoff_when_expire_ - Determines whether SMB client sessions with the SMB server will be forcibly disconnected when the client's logon hours expire + *forced* - keyword, number.long * _preferences.forced_ - 1 if the value is forced/managed, else 0 @@ -2250,7 +2300,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *hostname* - keyword, text.text -* _curl_certificate.hostname_ - Hostname to CURL (domain[:port], for example, osquery.io) +* _curl_certificate.hostname_ - Hostname to CURL (domain[:port], e.g. osquery.io) * _system_info.hostname_ - Network hostname including domain * _ycloud_instance_metadata.hostname_ - Hostname of the VM @@ -2626,7 +2676,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *is_active* - keyword, number.long -* _running_apps.is_active_ - 1 if the application is in focus, 0 otherwise +* _running_apps.is_active_ - (DEPRECATED) *is_hidden* - keyword, number.long @@ -2949,6 +2999,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _shared_memory.locked_ - 1 if segment is locked else 0 +*lockout_bad_count* - keyword, number.long + +* _security_profile_info.lockout_bad_count_ - Number of failed logon attempts after which a user account MUST be locked out + *log_file_disk_quota_mb* - keyword, number.long * _carbon_black_info.log_file_disk_quota_mb_ - Event file disk quota in MB @@ -2997,10 +3051,18 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _logon_sessions.logon_time_ - The time the session owner logged on. +*logon_to_change_password* - keyword, number.long + +* _security_profile_info.logon_to_change_password_ - Determines if logon session is required to change the password + *logon_type* - keyword, text.text * _logon_sessions.logon_type_ - The logon method. +*lsa_anonymous_name_lookup* - keyword, number.long + +* _security_profile_info.lsa_anonymous_name_lookup_ - Determines if an anonymous user is allowed to query the local LSA policy + *mac* - keyword, text.text * _arp_cache.mac_ - MAC address of broadcasted address @@ -3110,7 +3172,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *max_rows* - keyword, number.long -* _unified_log.max_rows_ - The max number of rows returned (defaults to 100). +* _unified_log.max_rows_ - the max number of rows returned (defaults to 100) *max_speed* - keyword, number.long @@ -3124,6 +3186,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _shared_resources.maximum_allowed_ - Limit on the maximum number of users allowed to use this resource concurrently. The value is only valid if the AllowMaximum property is set to FALSE. +*maximum_password_age* - keyword, number.long + +* _security_profile_info.maximum_password_age_ - Determines the maximum number of days that a password can be used before the client requires the user to change it + *md5* - keyword, text.text * _acpi_tables.md5_ - MD5 hash of table content @@ -3240,7 +3306,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _lxd_cluster_members.message_ - Message from the node (Online/Offline) * _selinux_events.message_ - Message * _syslog_events.message_ - The syslog message -* _unified_log.message_ - Composed message +* _unified_log.message_ - composed message * _user_events.message_ - Message from the event *metadata_endpoint* - keyword, text.text @@ -3297,6 +3363,14 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _memory_devices.min_voltage_ - Minimum operating voltage of device in millivolts +*minimum_password_age* - keyword, number.long + +* _security_profile_info.minimum_password_age_ - Determines the minimum number of days that a password must be used before the user can change it + +*minimum_password_length* - keyword, number.long + +* _security_profile_info.minimum_password_length_ - Determines the least number of characters that can make up a password for a user account + *minimum_system_version* - keyword, text.text * _apps.minimum_system_version_ - Minimum version of macOS required for the app to run @@ -3459,7 +3533,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _etc_protocols.name_ - Protocol name * _etc_services.name_ - Service name * _fan_speed_sensors.name_ - Fan name -* _fbsd_kmods.name_ - Module name * _firefox_addons.name_ - Addon display name * _homebrew_packages.name_ - Package name * _ie_extensions.name_ - Extension display name @@ -3491,7 +3564,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _package_install_history.name_ - Package display name * _physical_disk_performance.name_ - Name of the physical disk * _pipes.name_ - Name of the pipe -* _pkg_packages.name_ - Package name * _power_sensors.name_ - Name of power source * _processes.name_ - The process path or shorthand argv[0] * _programs.name_ - Commonly used product name. @@ -3529,7 +3601,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *native* - keyword, number.long * _browser_plugins.native_ - Plugin requires native execution -* _firefox_addons.native_ - 1 If the addon includes binary components else 0 *net_namespace* - keyword, text.text @@ -3561,6 +3632,14 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _docker_container_stats.network_tx_bytes_ - Total network bytes transmitted +*new_administrator_name* - keyword, text.text + +* _security_profile_info.new_administrator_name_ - Determines the name of the Administrator account on the local computer + +*new_guest_name* - keyword, text.text + +* _security_profile_info.new_guest_name_ - Determines the name of the Guest account on the local computer + *next_run_time* - keyword, number.long * _scheduled_tasks.next_run_time_ - Timestamp the task is scheduled to run next @@ -3916,6 +3995,14 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _wifi_networks.passpoint_ - 1 if Passpoint is supported, 0 otherwise +*password_complexity* - keyword, number.long + +* _security_profile_info.password_complexity_ - Determines whether passwords must meet a series of strong-password guidelines + +*password_history_size* - keyword, number.long + +* _security_profile_info.password_history_size_ - Number of unique new passwords that must be associated with a user account before an old password can be reused + *password_last_set_time* - keyword, number.double * _account_policy_data.password_last_set_time_ - The time the password was last changed @@ -4150,10 +4237,10 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _processes.pid_ - Process (or thread) ID * _running_apps.pid_ - The pid of the application * _seccomp_events.pid_ - Process ID -* _services.pid_ - The Process ID of the service +* _services.pid_ - the Process ID of the service * _shared_memory.pid_ - Process ID to last use the segment * _socket_events.pid_ - Process (or thread) ID -* _unified_log.pid_ - The pid of the process that made the entry +* _unified_log.pid_ - the pid of the process that made the entry * _user_events.pid_ - Process (or thread) ID * _windows_crashes.pid_ - Process ID of the crashed process * _windows_eventlog.pid_ - Process ID which emitted the event record @@ -4327,7 +4414,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *process* - keyword, text.text * _alf_explicit_auths.process_ - Process name explicitly allowed -* _unified_log.process_ - The name of the process that made the entry +* _unified_log.process_ - the name of the process that made the entry *process_being_tapped* - keyword, number.long @@ -4560,7 +4647,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *refs* - keyword, number.long -* _fbsd_kmods.refs_ - Module reverse dependencies * _kernel_extensions.refs_ - Reference count *region* - keyword, text.text @@ -4875,7 +4961,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *sender* - keyword, text.text * _asl.sender_ - Sender's identification string. Default is process name. -* _unified_log.sender_ - The name of the binary image that made the entry +* _unified_log.sender_ - the name of the binary image that made the entry *sensor_backend_server* - keyword, text.text @@ -5101,7 +5187,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _device_file.size_ - Size of file in bytes * _disk_events.size_ - Size of partition in bytes * _docker_image_history.size_ - Size of instruction in bytes -* _fbsd_kmods.size_ - Size of module content * _file.size_ - Size of file in bytes * _file_events.size_ - Size of file in bytes * _kernel_extensions.size_ - Bytes of wired memory used by extension @@ -5337,7 +5422,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *storage* - keyword, number.long -* _unified_log.storage_ - The storage category for the entry. +* _unified_log.storage_ - the storage category for the entry *storage_driver* - keyword, text.text @@ -5416,7 +5501,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *subsystem* - keyword, text.text * _system_controls.subsystem_ - Subsystem ID, control type -* _unified_log.subsystem_ - The subsystem of the os_log_t used +* _unified_log.subsystem_ - the subsystem of the os_log_t used *subsystem_model* - keyword, text.text @@ -5585,7 +5670,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _bpf_process_events.tid_ - Thread ID * _bpf_socket_events.tid_ - Thread ID -* _unified_log.tid_ - The tid of the thread that made the entry +* _unified_log.tid_ - the tid of the thread that made the entry * _windows_crashes.tid_ - Thread ID of the crashed thread * _windows_eventlog.tid_ - Thread ID which emitted the event record @@ -5637,7 +5722,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *timestamp* - keyword, text.text * _time.timestamp_ - Current timestamp (log format) in UTC -* _unified_log.timestamp_ - Unix timestamp associated with the entry +* _unified_log.timestamp_ - unix timestamp associated with the entry * _windows_eventlog.timestamp_ - Timestamp to selectively filter the events *timestamp_ms* - keyword, number.long @@ -6078,7 +6163,6 @@ For more information about osquery tables, see the https://osquery.io/schema[osq * _osquery_packs.version_ - Minimum osquery version that this query will run on * _package_install_history.version_ - Package display version * _package_receipts.version_ - Installed package version -* _pkg_packages.version_ - Package version * _platform_info.version_ - Platform code version * _portage_keywords.version_ - The version which are affected by the use flags, empty means all * _portage_packages.version_ - The version which are affected by the use flags, empty means all From 551f0f9e5c9b5c48613004976c6ed89989936448 Mon Sep 17 00:00:00 2001 From: Drew Tate Date: Mon, 6 Feb 2023 13:11:41 -0600 Subject: [PATCH 007/134] [Lens] don't block render on missing sort field (#150356) --- .../form_based/operations/definitions/last_value.test.tsx | 3 --- .../form_based/operations/definitions/last_value.tsx | 1 - 2 files changed, 4 deletions(-) diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.test.tsx index be43974b7ba44..3471c82a43fa4 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.test.tsx @@ -980,9 +980,6 @@ describe('last_value', () => { "dimensionId": "col1", "id": "dimensionButton", }, - Object { - "id": "visualization", - }, Object { "id": "embeddableBadge", }, diff --git a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx index 46a9335f20d1c..1f0e50ecb2ac6 100644 --- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/last_value.tsx @@ -86,7 +86,6 @@ function getInvalidSortFieldMessage( displayLocations: [ { id: 'toolbar' }, { id: 'dimensionButton', dimensionId: columnId }, - { id: 'visualization' }, { id: 'embeddableBadge' }, ], }; From a290e4038fee6e25dc3674dae4861e875be81465 Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Mon, 6 Feb 2023 20:14:09 +0100 Subject: [PATCH 008/134] [Enterprise Search] Add toggle for HTML extraction (#150352) ## Summary This adds a toggle to enable/disable full html extraction for Elastic crawlers. https://user-images.githubusercontent.com/94373878/217030130-8b377aad-dbbf-4b55-87ae-b0a94ac060a6.mov --- .../common/types/connectors.ts | 4 +- .../update_html_extraction_api_logic.test.ts | 36 ++++++ .../update_html_extraction_api_logic.ts | 44 +++++++ .../crawler_configuration.tsx | 107 ++++++++++++++++++ .../crawler_configuration_logic.ts | 85 ++++++++++++++ .../search_index/index_view_logic.ts | 6 + .../components/search_index/search_index.tsx | 12 ++ .../lib/crawler/put_html_extraction.test.ts | 51 +++++++++ .../server/lib/crawler/put_html_extraction.ts | 32 ++++++ .../enterprise_search/crawler/crawler.ts | 40 +++++++ 10 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration_logic.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.test.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.ts diff --git a/x-pack/plugins/enterprise_search/common/types/connectors.ts b/x-pack/plugins/enterprise_search/common/types/connectors.ts index 87ccbad824e1f..78f3f9e6a51e4 100644 --- a/x-pack/plugins/enterprise_search/common/types/connectors.ts +++ b/x-pack/plugins/enterprise_search/common/types/connectors.ts @@ -10,7 +10,9 @@ export interface KeyValuePair { value: string | null; } -export type ConnectorConfiguration = Record; +export type ConnectorConfiguration = Record & { + extract_full_html?: { label: string; value: boolean }; +}; export interface ConnectorScheduling { enabled: boolean; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.test.ts new file mode 100644 index 0000000000000..163a29e07f42e --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockHttpValues } from '../../../__mocks__/kea_logic'; + +import { updateHtmlExtraction } from './update_html_extraction_api_logic'; + +describe('UpdateHtmlExtractionApiLogic', () => { + const { http } = mockHttpValues; + beforeEach(() => { + jest.clearAllMocks(); + }); + describe('updateHtmlExtraction', () => { + it('calls correct api', async () => { + const indexName = 'elastic-co-crawler'; + + http.get.mockReturnValue(Promise.resolve()); + + const result = updateHtmlExtraction({ htmlExtraction: true, indexName }); + + expect(http.put).toHaveBeenCalledWith( + `/internal/enterprise_search/indices/${indexName}/crawler/html_extraction`, + { + body: JSON.stringify({ + extract_full_html: true, + }), + } + ); + await expect(result).resolves.toEqual({ htmlExtraction: true }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.ts new file mode 100644 index 0000000000000..e6ddcc5f82071 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/update_html_extraction_api_logic.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Actions } from '../../../shared/api_logic/create_api_logic'; + +import { createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { HttpLogic } from '../../../shared/http'; + +export interface UpdateHtmlExtractionArgs { + htmlExtraction: boolean; + indexName: string; +} + +export interface UpdateHtmlExtractionResponse { + htmlExtraction: boolean; +} + +export const updateHtmlExtraction = async ({ + htmlExtraction, + indexName, +}: UpdateHtmlExtractionArgs) => { + const route = `/internal/enterprise_search/indices/${indexName}/crawler/html_extraction`; + + const params = { extract_full_html: htmlExtraction }; + + await HttpLogic.values.http.put(route, { + body: JSON.stringify(params), + }); + return { htmlExtraction }; +}; + +export const UpdateHtmlExtractionApiLogic = createApiLogic( + ['update_html_extraction_api_logic'], + updateHtmlExtraction +); + +export type UpdateHtmlExtractionActions = Actions< + UpdateHtmlExtractionArgs, + UpdateHtmlExtractionResponse +>; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration.tsx new file mode 100644 index 0000000000000..4741525a85b72 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration.tsx @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useActions, useValues } from 'kea'; + +import { + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiSpacer, + EuiSplitPanel, + EuiSwitch, + EuiText, + EuiTitle, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import { Status } from '../../../../../../../common/types/api'; + +import { IndexViewLogic } from '../../index_view_logic'; + +import { CrawlerConfigurationLogic } from './crawler_configuration_logic'; + +export const CrawlerConfiguration: React.FC = () => { + const { htmlExtraction } = useValues(IndexViewLogic); + const { status } = useValues(CrawlerConfigurationLogic); + const { updateHtmlExtraction } = useActions(CrawlerConfigurationLogic); + return ( + <> + + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.crawler.crawlerConfiguration.extractHTML.title', + { defaultMessage: 'Store full HTML' } + )} +

+
+ + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.crawler.crawlerConfiguration.extractHTML.addExtraFieldDescription', + { + defaultMessage: + 'Add an extra field in all documents with the value of the full HTML of the page being crawled.', + } + )} +

+
+ + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.crawler.crawlerConfiguration.extractHTML.increasedSizeWarning', + { + defaultMessage: + 'This may dramatically increase the index size if the site being crawled is large.', + } + )} +

+
+
+ + + + updateHtmlExtraction(event.target.checked)} + /> + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.crawler.crawlerConfiguration.extractHTML.learnMoreLink', + { + defaultMessage: 'Learn more about storing full HTML.', + } + )} + + + + +
+ + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration_logic.ts new file mode 100644 index 0000000000000..9298ca3b32585 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_configuration/crawler_configuration_logic.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { Status } from '../../../../../../../common/types/api'; +import { Connector } from '../../../../../../../common/types/connectors'; + +import { + UpdateHtmlExtractionActions, + UpdateHtmlExtractionApiLogic, +} from '../../../../api/crawler/update_html_extraction_api_logic'; +import { CachedFetchIndexApiLogicActions } from '../../../../api/index/cached_fetch_index_api_logic'; +import { isCrawlerIndex } from '../../../../utils/indices'; +import { IndexViewLogic } from '../../index_view_logic'; + +interface CrawlerConfigurationLogicActions { + apiError: UpdateHtmlExtractionActions['apiError']; + apiSuccess: UpdateHtmlExtractionActions['apiSuccess']; + fetchIndex: () => void; + fetchIndexApiSuccess: CachedFetchIndexApiLogicActions['apiSuccess']; + htmlExtraction: boolean; + makeRequest: UpdateHtmlExtractionActions['makeRequest']; + updateHtmlExtraction(htmlExtraction: boolean): { htmlExtraction: boolean }; +} + +interface CrawlerConfigurationLogicValues { + connector: Connector | undefined; + indexName: string; + localHtmlExtraction: boolean | null; + status: Status; +} + +export const CrawlerConfigurationLogic = kea< + MakeLogicType +>({ + actions: { + updateHtmlExtraction: (htmlExtraction) => ({ htmlExtraction }), + }, + connect: { + actions: [ + IndexViewLogic, + ['fetchIndex', 'fetchIndexApiSuccess'], + UpdateHtmlExtractionApiLogic, + ['apiSuccess', 'makeRequest'], + ], + values: [IndexViewLogic, ['connector', 'indexName'], UpdateHtmlExtractionApiLogic, ['status']], + }, + listeners: ({ actions, values }) => ({ + apiSuccess: () => { + actions.fetchIndex(); + }, + updateHtmlExtraction: ({ htmlExtraction }) => { + actions.makeRequest({ htmlExtraction, indexName: values.indexName }); + }, + }), + path: ['enterprise_search', 'search_index', 'crawler', 'configuration'], + reducers: { + localHtmlExtraction: [ + null, + { + apiSuccess: (_, { htmlExtraction }) => htmlExtraction, + fetchIndexApiSuccess: (_, index) => { + if (isCrawlerIndex(index)) { + return index.connector.configuration.extract_full_html?.value ?? null; + } + return null; + }, + }, + ], + }, + selectors: ({ selectors }) => ({ + htmlExtraction: [ + () => [selectors.connector, selectors.localHtmlExtraction], + (connector: Connector | null, localHtmlExtraction: boolean | null) => + localHtmlExtraction !== null + ? localHtmlExtraction + : connector?.configuration.extract_full_html?.value ?? false, + ], + }), +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts index 26c1f823b479d..4ed5b7cef1b41 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts @@ -70,6 +70,7 @@ export interface IndexViewValues { hasAdvancedFilteringFeature: boolean; hasBasicFilteringFeature: boolean; hasFilteringFeature: boolean; + htmlExtraction: boolean | undefined; index: ElasticsearchViewIndex | undefined; indexData: typeof CachedFetchIndexApiLogic.values.indexData; indexName: string; @@ -214,6 +215,11 @@ export const IndexViewLogic = kea [selectors.hasAdvancedFilteringFeature, selectors.hasBasicFilteringFeature], (advancedFeature: boolean, basicFeature: boolean) => advancedFeature || basicFeature, ], + htmlExtraction: [ + () => [selectors.connector], + (connector: Connector | undefined) => + connector?.configuration.extract_full_html?.value ?? undefined, + ], index: [ () => [selectors.indexData], (data: IndexViewValues['indexData']) => (data ? indexToViewIndex(data) : undefined), diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx index 5c5b691d42ce1..b09a0a9d8a96a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx @@ -36,6 +36,7 @@ import { ConnectorSchedulingComponent } from './connector/connector_scheduling'; import { ConnectorSyncRules } from './connector/sync_rules/connector_rules'; import { AutomaticCrawlScheduler } from './crawler/automatic_crawl_scheduler/automatic_crawl_scheduler'; import { CrawlCustomSettingsFlyout } from './crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout'; +import { CrawlerConfiguration } from './crawler/crawler_configuration/crawler_configuration'; import { SearchIndexDomainManagement } from './crawler/domain_management/domain_management'; import { SearchIndexDocuments } from './documents'; import { SearchIndexIndexMappings } from './index_mappings'; @@ -56,6 +57,7 @@ export enum SearchIndexTabId { SCHEDULING = 'scheduling', // crawler indices DOMAIN_MANAGEMENT = 'domain_management', + CRAWLER_CONFIGURATION = 'crawler_configuration', } export const SearchIndex: React.FC = () => { @@ -164,6 +166,16 @@ export const SearchIndex: React.FC = () => { defaultMessage: 'Manage Domains', }), }, + { + content: , + id: SearchIndexTabId.CRAWLER_CONFIGURATION, + name: i18n.translate( + 'xpack.enterpriseSearch.content.searchIndex.crawlerConfigurationTabLabel', + { + defaultMessage: 'Configuration', + } + ), + }, { content: , id: SearchIndexTabId.SCHEDULING, diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.test.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.test.ts new file mode 100644 index 0000000000000..d859d9639b2b6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core/server'; + +import { CONNECTORS_INDEX } from '../..'; +import { Connector } from '../../../common/types/connectors'; + +import { updateHtmlExtraction } from './put_html_extraction'; + +describe('updateHtmlExtraction lib function', () => { + const mockClient = { + asCurrentUser: { + update: jest.fn(), + }, + asInternalUser: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should update connector configuration', async () => { + mockClient.asCurrentUser.update.mockResolvedValue(true); + const mockConnector = { + configuration: { test: { label: 'haha', value: 'this' } }, + id: 'connectorId', + }; + + await updateHtmlExtraction( + mockClient as unknown as IScopedClusterClient, + true, + mockConnector as any as Connector + ); + expect(mockClient.asCurrentUser.update).toHaveBeenCalledWith({ + doc: { + configuration: { + ...mockConnector.configuration, + extract_full_html: { label: 'Extract full HTML', value: true }, + }, + }, + id: 'connectorId', + index: CONNECTORS_INDEX, + refresh: 'wait_for', + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.ts new file mode 100644 index 0000000000000..bf7c18a575853 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/put_html_extraction.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +import { CONNECTORS_INDEX } from '../..'; +import { Connector } from '../../../common/types/connectors'; + +export async function updateHtmlExtraction( + client: IScopedClusterClient, + htmlExtraction: boolean, + connector: Connector +) { + return await client.asCurrentUser.update({ + doc: { + configuration: { + ...connector.configuration, + extract_full_html: { + label: 'Extract full HTML', + value: htmlExtraction, + }, + }, + }, + id: connector.id, + index: CONNECTORS_INDEX, + refresh: 'wait_for', + }); +} diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts index c3b034f0b6ce7..08cea961709fc 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts @@ -6,6 +6,7 @@ */ import { schema } from '@kbn/config-schema'; + import { i18n } from '@kbn/i18n'; import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../../common/constants'; @@ -15,6 +16,7 @@ import { addConnector } from '../../../lib/connectors/add_connector'; import { deleteConnectorById } from '../../../lib/connectors/delete_connector'; import { fetchConnectorByIndexName } from '../../../lib/connectors/fetch_connectors'; import { fetchCrawlerByIndexName } from '../../../lib/crawler/fetch_crawlers'; +import { updateHtmlExtraction } from '../../../lib/crawler/put_html_extraction'; import { deleteIndex } from '../../../lib/indices/delete_index'; import { RouteDependencies } from '../../../plugin'; import { createError } from '../../../utils/create_error'; @@ -389,6 +391,44 @@ export function registerCrawlerRoutes(routeDependencies: RouteDependencies) { }) ); + router.put( + { + path: '/internal/enterprise_search/indices/{indexName}/crawler/html_extraction', + validate: { + body: schema.object({ + extract_full_html: schema.boolean(), + }), + params: schema.object({ + indexName: schema.string(), + }), + }, + }, + elasticsearchErrorHandler(log, async (context, request, response) => { + const { client } = (await context.core).elasticsearch; + + const connector = await fetchConnectorByIndexName(client, request.params.indexName); + if ( + connector && + connector.service_type === ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE + ) { + await updateHtmlExtraction(client, request.body.extract_full_html, connector); + return response.ok(); + } else { + return createError({ + errorCode: ErrorCode.RESOURCE_NOT_FOUND, + message: i18n.translate( + 'xpack.enterpriseSearch.server.routes.updateHtmlExtraction.noCrawlerFound', + { + defaultMessage: 'Could not find a crawler for this index', + } + ), + response, + statusCode: 404, + }); + } + }) + ); + registerCrawlerCrawlRulesRoutes(routeDependencies); registerCrawlerEntryPointRoutes(routeDependencies); registerCrawlerSitemapRoutes(routeDependencies); From 6e36fdbe53451fa4ca8cbf86302a931f73b3d977 Mon Sep 17 00:00:00 2001 From: Kfir Peled <61654899+kfirpeled@users.noreply.github.com> Date: Mon, 6 Feb 2023 21:44:15 +0200 Subject: [PATCH 009/134] [Cloud Security] added `/internal/cloud_security_posture/status` basic tests (#150102) --- .../plugins/cloud_security_posture/README.md | 5 + .../apis/cloud_security_posture/index.ts | 1 + .../apis/cloud_security_posture/status.ts | 125 ++++++++++++++++++ x-pack/test/tsconfig.json | 1 + 4 files changed, 132 insertions(+) create mode 100644 x-pack/test/api_integration/apis/cloud_security_posture/status.ts diff --git a/x-pack/plugins/cloud_security_posture/README.md b/x-pack/plugins/cloud_security_posture/README.md index 7a4646d08ba07..a655d292c39ee 100755 --- a/x-pack/plugins/cloud_security_posture/README.md +++ b/x-pack/plugins/cloud_security_posture/README.md @@ -49,3 +49,8 @@ yarn test:ftr --config x-pack/test/cloud_security_posture_functional/config.ts > **Note** > in development, run them separately with `ftr:runner` and `ftr:server` + +```bash +yarn test:ftr:server --config x-pack/test/api_integration/config.ts +yarn test:ftr:runner --include-tag=cloud_security_posture --config x-pack/test/api_integration/config.ts +``` \ No newline at end of file diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/index.ts b/x-pack/test/api_integration/apis/cloud_security_posture/index.ts index 8b80e0d0eab0c..60dc074044ba1 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/index.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/index.ts @@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('cloud_security_posture', function () { this.tags(['cloud_security_posture']); + loadTestFile(require.resolve('./status')); // Place your tests files under this directory and add the following here: // loadTestFile(require.resolve('./your test name')); diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts new file mode 100644 index 0000000000000..6d10aa2f60f4a --- /dev/null +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import type { CspSetupStatus } from '@kbn/cloud-security-posture-plugin/common/types'; +import type { SuperTest, Test } from 'supertest'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + + describe('GET /internal/cloud_security_posture/status', () => { + let agentPolicyId: string; + + beforeEach(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + + const { body: agentPolicyResponse } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + }); + agentPolicyId = agentPolicyResponse.item.id; + }); + + afterEach(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + }); + + it(`Should return not-deployed when installed kspm`, async () => { + await createPackagePolicy( + supertest, + agentPolicyId, + 'kspm', + 'cloudbeat/cis_k8s', + 'vanilla', + 'kspm' + ); + + const { body: res }: { body: CspSetupStatus } = await supertest + .get(`/internal/cloud_security_posture/status`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + expect(res.status).to.be('not-deployed'); + expect(res.installedPolicyTemplates).length(1).contain('kspm'); + expect(res.healthyAgents).to.be(0); + }); + + it(`Should return not-deployed when installed cspm`, async () => { + await createPackagePolicy( + supertest, + agentPolicyId, + 'cspm', + 'cloudbeat/cis_aws', + 'aws', + 'cspm' + ); + + const { body: res }: { body: CspSetupStatus } = await supertest + .get(`/internal/cloud_security_posture/status`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + expect(res.status).to.be('not-deployed'); + expect(res.installedPolicyTemplates).length(1).contain('cspm'); + expect(res.healthyAgents).to.be(0); + }); + }); +} + +async function createPackagePolicy( + supertest: SuperTest, + agentPolicyId: string, + policyTemplate: string, + input: string, + deployment: string, + posture: string +) { + const { body: postPackageResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + force: true, + name: 'cloud_security_posture-1', + description: '', + namespace: 'default', + policy_id: agentPolicyId, + enabled: true, + inputs: [ + { + enabled: true, + type: input, + policy_template: policyTemplate, + }, + ], + package: { + name: 'cloud_security_posture', + title: 'Kubernetes Security Posture Management', + version: '1.2.8', + }, + vars: { + deployment: { + value: deployment, + type: 'text', + }, + posture: { + value: posture, + type: 'text', + }, + }, + }) + .expect(200); + + return postPackageResponse.item; +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 43f96bcd0c429..ef8c79b7b03ca 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -112,5 +112,6 @@ "@kbn/stdio-dev-helpers", "@kbn/alerting-api-integration-helpers", "@kbn/securitysolution-ecs", + "@kbn/cloud-security-posture-plugin", ] } From 78947c4c6f0638d25f38904d967b922aa1f99726 Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 6 Feb 2023 13:52:01 -0600 Subject: [PATCH 010/134] Upgrade terser to 5.16.1 (#149702) Also updates our renovate config to track this. Plan on adding a few more build related dependencies. --- package.json | 2 +- renovate.json | 11 +++++++++++ yarn.lock | 8 ++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 049163e6dfd7c..c2613ebda6e76 100644 --- a/package.json +++ b/package.json @@ -1176,7 +1176,7 @@ "svgo": "^2.8.0", "tape": "^5.0.1", "tempy": "^0.3.0", - "terser": "^5.15.1", + "terser": "^5.16.1", "terser-webpack-plugin": "^4.2.3", "tough-cookie": "^4.1.2", "tree-kill": "^1.2.2", diff --git a/renovate.json b/renovate.json index 932f0a6558864..ce12b9dd5a248 100644 --- a/renovate.json +++ b/renovate.json @@ -153,6 +153,17 @@ "labels": ["Team:Operations", "release_note:skip", "backport:all-open"], "enabled": true }, + { + "groupName": "minify", + "packageNames": [ + "gulp-terser", + "terser" + ], + "reviewers": ["team:kibana-operations"], + "matchBaseBranches": ["main"], + "labels": ["Team:Operations", "release_note:skip"], + "enabled": true + }, { "groupName": "@testing-library", "packageNames": [ diff --git a/yarn.lock b/yarn.lock index fd2cb119a8198..77b529bdd0bcb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26175,10 +26175,10 @@ terser@^4.1.2, terser@^4.6.3: source-map "~0.6.1" source-map-support "~0.5.12" -terser@^5.14.1, terser@^5.15.1, terser@^5.3.4, terser@^5.9.0: - version "5.15.1" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.15.1.tgz#8561af6e0fd6d839669c73b92bdd5777d870ed6c" - integrity sha512-K1faMUvpm/FBxjBXud0LWVAGxmvoPbZbfTCYbSgaaYQaIXI3/TdI7a7ZGA73Zrou6Q8Zmz3oeUTsp/dj+ag2Xw== +terser@^5.14.1, terser@^5.16.1, terser@^5.3.4, terser@^5.9.0: + version "5.16.1" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.16.1.tgz#5af3bc3d0f24241c7fb2024199d5c461a1075880" + integrity sha512-xvQfyfA1ayT0qdK47zskQgRZeWLoOQ8JQ6mIgRGVNwZKdQMU+5FkCBjmv4QjcrTzyZquRw2FVtlJSRUmMKQslw== dependencies: "@jridgewell/source-map" "^0.3.2" acorn "^8.5.0" From ebc1bc5242a317ad1b3fa92a80373590580ee3b2 Mon Sep 17 00:00:00 2001 From: Drew Tate Date: Mon, 6 Feb 2023 13:53:56 -0600 Subject: [PATCH 011/134] [Lens] suppress missing index toasts (#150345) --- x-pack/plugins/lens/public/data_views_service/loader.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/data_views_service/loader.ts b/x-pack/plugins/lens/public/data_views_service/loader.ts index abd8a48815122..bb27eec2076e3 100644 --- a/x-pack/plugins/lens/public/data_views_service/loader.ts +++ b/x-pack/plugins/lens/public/data_views_service/loader.ts @@ -176,7 +176,9 @@ export async function loadIndexPatterns({ } indexPatterns.push( ...(await Promise.all( - Object.values(adHocDataViews || {}).map((spec) => dataViews.create(spec)) + Object.values(adHocDataViews || {}).map((spec) => + dataViews.create({ ...spec, allowNoIndex: true }) + ) )) ); From 148a49adb8378afc03f3b8d35fad4393a41def87 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 6 Feb 2023 21:14:51 +0100 Subject: [PATCH 012/134] [Console] Replace global `GET /_mapping` request with `GET /_mapping` (#147770) ## Summary ### Notes for reviewers - Currently, autocomplete suggestions for fields don't work with wildcards and data streams due to the [bug](https://github.com/elastic/kibana/issues/149496) in the `main`. It should be addressed separately. ### How to test In order to spot the loading behaviour, ideally you should create an index with a heavy mappings definition. Afterwards, write a query that requires a field from this index, e.g.: ``` GET /_search { "aggs": { "my_agg": { "terms": { "field": "", "size": 10 } } } } ``` Place a cursor next to the `field` property, it should trigger mappings fetch. After that, the mappings definition for this index will be cached and accessed synchronously. You can also open the browser's dev tools and enable Network throttling. It allows noticing loading behaviour for any index. -------------------- Resolves https://github.com/elastic/kibana/issues/146855 Instead of fetching all mappings upfront, requests mapping definition on demand per index according to the cursor position. Considering there is a maximum response size limit of 10MB in the `/autocomplete_entities` endpoint, field autocompletion wasn't working at all if the overall mappings definition exceeded this size. Retrieving mappings per index tackles this and improves the init time. ![Jan-25-2023 17-16-31](https://user-images.githubusercontent.com/5236598/214616790-4954d005-e56f-49f9-be6d-435c076270a8.gif) ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --------- Co-authored-by: Jean-Louis Leysens --- .../application/containers/editor/editor.tsx | 15 +- .../console/public/application/index.tsx | 2 + .../legacy_core_editor/legacy_core_editor.ts | 30 +- .../models/sense_editor/integration.test.js | 15 +- .../public/lib/autocomplete/autocomplete.ts | 175 ++++++---- .../console/public/lib/autocomplete/types.ts | 9 + .../autocomplete_entities.test.js | 318 +++++++++++------- .../lib/autocomplete_entities/mapping.ts | 121 ++++++- .../console/public/services/autocomplete.ts | 7 +- .../console/public/services/settings.ts | 7 +- .../console/public/types/core_editor.ts | 7 +- .../console/autocomplete_entities/index.ts | 36 +- .../validation_config.ts | 33 ++ src/plugins/console/tsconfig.json | 1 + .../apis/console/autocomplete_entities.ts | 12 +- 15 files changed, 554 insertions(+), 234 deletions(-) create mode 100644 src/plugins/console/server/routes/api/console/autocomplete_entities/validation_config.ts diff --git a/src/plugins/console/public/application/containers/editor/editor.tsx b/src/plugins/console/public/application/containers/editor/editor.tsx index 93c11aa53c602..788039a2dc606 100644 --- a/src/plugins/console/public/application/containers/editor/editor.tsx +++ b/src/plugins/console/public/application/containers/editor/editor.tsx @@ -6,14 +6,14 @@ * Side Public License, v 1. */ -import React, { useCallback, memo } from 'react'; +import React, { useCallback, memo, useEffect, useState } from 'react'; import { debounce } from 'lodash'; import { EuiProgress } from '@elastic/eui'; import { EditorContentSpinner } from '../../components'; import { Panel, PanelsContainer } from '..'; import { Editor as EditorUI, EditorOutput } from './legacy/console_editor'; -import { StorageKeys } from '../../../services'; +import { getAutocompleteInfo, StorageKeys } from '../../../services'; import { useEditorReadContext, useServicesContext, useRequestReadContext } from '../../contexts'; import type { SenseEditor } from '../../models'; @@ -33,6 +33,15 @@ export const Editor = memo(({ loading, setEditorInstance }: Props) => { const { currentTextObject } = useEditorReadContext(); const { requestInFlight } = useRequestReadContext(); + const [fetchingMappings, setFetchingMappings] = useState(false); + + useEffect(() => { + const subscription = getAutocompleteInfo().mapping.isLoading$.subscribe(setFetchingMappings); + return () => { + subscription.unsubscribe(); + }; + }, []); + const [firstPanelWidth, secondPanelWidth] = storage.get(StorageKeys.WIDTH, [ INITIAL_PANEL_WIDTH, INITIAL_PANEL_WIDTH, @@ -50,7 +59,7 @@ export const Editor = memo(({ loading, setEditorInstance }: Props) => { return ( <> - {requestInFlight ? ( + {requestInFlight || fetchingMappings ? (
diff --git a/src/plugins/console/public/application/index.tsx b/src/plugins/console/public/application/index.tsx index 1cf9a54210973..92d875b9db2a9 100644 --- a/src/plugins/console/public/application/index.tsx +++ b/src/plugins/console/public/application/index.tsx @@ -69,6 +69,8 @@ export function renderApp({ const api = createApi({ http }); const esHostService = createEsHostService({ api }); + autocompleteInfo.mapping.setup(http, settings); + render( diff --git a/src/plugins/console/public/application/models/legacy_core_editor/legacy_core_editor.ts b/src/plugins/console/public/application/models/legacy_core_editor/legacy_core_editor.ts index 5def8a696df2f..5c041a95e216f 100644 --- a/src/plugins/console/public/application/models/legacy_core_editor/legacy_core_editor.ts +++ b/src/plugins/console/public/application/models/legacy_core_editor/legacy_core_editor.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import ace from 'brace'; +import ace, { type Annotation } from 'brace'; import { Editor as IAceEditor, IEditSession as IAceEditSession } from 'brace'; import $ from 'jquery'; import { @@ -402,8 +402,7 @@ export class LegacyCoreEditor implements CoreEditor { getCompletions: ( // eslint-disable-next-line @typescript-eslint/naming-convention DO_NOT_USE_1: IAceEditor, - // eslint-disable-next-line @typescript-eslint/naming-convention - DO_NOT_USE_2: IAceEditSession, + aceEditSession: IAceEditSession, pos: { row: number; column: number }, prefix: string, callback: (...args: unknown[]) => void @@ -412,7 +411,30 @@ export class LegacyCoreEditor implements CoreEditor { lineNumber: pos.row + 1, column: pos.column + 1, }; - autocompleter(position, prefix, callback); + + const getAnnotationControls = () => { + let customAnnotation: Annotation; + return { + setAnnotation(text: string) { + const annotations = aceEditSession.getAnnotations(); + customAnnotation = { + text, + row: pos.row, + column: pos.column, + type: 'warning', + }; + + aceEditSession.setAnnotations([...annotations, customAnnotation]); + }, + removeAnnotation() { + aceEditSession.setAnnotations( + aceEditSession.getAnnotations().filter((a: Annotation) => a !== customAnnotation) + ); + }, + }; + }; + + autocompleter(position, prefix, callback, getAnnotationControls()); }, }, ]); diff --git a/src/plugins/console/public/application/models/sense_editor/integration.test.js b/src/plugins/console/public/application/models/sense_editor/integration.test.js index 434deb5456caf..08912871052ba 100644 --- a/src/plugins/console/public/application/models/sense_editor/integration.test.js +++ b/src/plugins/console/public/application/models/sense_editor/integration.test.js @@ -13,6 +13,9 @@ import $ from 'jquery'; import * as kb from '../../../lib/kb/kb'; import { AutocompleteInfo, setAutocompleteInfo } from '../../../services'; +import { httpServiceMock } from '@kbn/core-http-browser-mocks'; +import { StorageMock } from '../../../services/storage.mock'; +import { SettingsMock } from '../../../services/settings.mock'; describe('Integration', () => { let senseEditor; @@ -27,6 +30,15 @@ describe('Integration', () => { $(senseEditor.getCoreEditor().getContainer()).show(); senseEditor.autocomplete._test.removeChangeListener(); autocompleteInfo = new AutocompleteInfo(); + + const httpMock = httpServiceMock.createSetupContract(); + const storage = new StorageMock({}, 'test'); + const settingsMock = new SettingsMock(storage); + + settingsMock.getAutocomplete.mockReturnValue({ fields: true }); + + autocompleteInfo.mapping.setup(httpMock, settingsMock); + setAutocompleteInfo(autocompleteInfo); }); afterEach(() => { @@ -164,7 +176,8 @@ describe('Integration', () => { ac('textBoxPosition', posCompare); ac('rangeToReplace', rangeCompare); done(); - } + }, + { setAnnotation: () => {}, removeAnnotation: () => {} } ); }); }); diff --git a/src/plugins/console/public/lib/autocomplete/autocomplete.ts b/src/plugins/console/public/lib/autocomplete/autocomplete.ts index 4e3770779f580..b24767b361d7e 100644 --- a/src/plugins/console/public/lib/autocomplete/autocomplete.ts +++ b/src/plugins/console/public/lib/autocomplete/autocomplete.ts @@ -593,7 +593,10 @@ export default function ({ return null; } - if (!context.autoCompleteSet) { + const isMappingsFetchingInProgress = + context.autoCompleteType === 'body' && !!context.asyncResultsState?.isLoading; + + if (!context.autoCompleteSet && !isMappingsFetchingInProgress) { return null; // nothing to do.. } @@ -1123,80 +1126,112 @@ export default function ({ } } + /** + * Extracts terms from the autocomplete set. + * @param context + */ + function getTerms(context: AutoCompleteContext, autoCompleteSet: ResultTerm[]) { + const terms = _.map( + autoCompleteSet.filter((term) => Boolean(term) && term.name != null), + function (term) { + if (typeof term !== 'object') { + term = { + name: term, + }; + } else { + term = _.clone(term); + } + const defaults: { + value?: string; + meta: string; + score: number; + context: AutoCompleteContext; + completer?: { insertMatch: (v: unknown) => void }; + } = { + value: term.name, + meta: 'API', + score: 0, + context, + }; + // we only need our custom insertMatch behavior for the body + if (context.autoCompleteType === 'body') { + defaults.completer = { + insertMatch() { + return applyTerm(term); + }, + }; + } + return _.defaults(term, defaults); + } + ); + + terms.sort(function ( + t1: { score: number; name?: string }, + t2: { score: number; name?: string } + ) { + /* score sorts from high to low */ + if (t1.score > t2.score) { + return -1; + } + if (t1.score < t2.score) { + return 1; + } + /* names sort from low to high */ + if (t1.name! < t2.name!) { + return -1; + } + if (t1.name === t2.name) { + return 0; + } + return 1; + }); + + return terms; + } + + function getSuggestions(terms: ResultTerm[]) { + return _.map(terms, function (t, i) { + t.insertValue = t.insertValue || t.value; + t.value = '' + t.value; // normalize to strings + t.score = -i; + return t; + }); + } + function getCompletions( position: Position, prefix: string, - callback: (e: Error | null, result: ResultTerm[] | null) => void + callback: (e: Error | null, result: ResultTerm[] | null) => void, + annotationControls: { + setAnnotation: (text: string) => void; + removeAnnotation: () => void; + } ) { try { const context = getAutoCompleteContext(editor, position); + if (!context) { callback(null, []); } else { - const terms = _.map( - context.autoCompleteSet!.filter((term) => Boolean(term) && term.name != null), - function (term) { - if (typeof term !== 'object') { - term = { - name: term, - }; - } else { - term = _.clone(term); - } - const defaults: { - value?: string; - meta: string; - score: number; - context: AutoCompleteContext; - completer?: { insertMatch: (v: unknown) => void }; - } = { - value: term.name, - meta: 'API', - score: 0, - context, - }; - // we only need our custom insertMatch behavior for the body - if (context.autoCompleteType === 'body') { - defaults.completer = { - insertMatch() { - return applyTerm(term); - }, - }; - } - return _.defaults(term, defaults); - } - ); - - terms.sort(function ( - t1: { score: number; name?: string }, - t2: { score: number; name?: string } - ) { - /* score sorts from high to low */ - if (t1.score > t2.score) { - return -1; - } - if (t1.score < t2.score) { - return 1; - } - /* names sort from low to high */ - if (t1.name! < t2.name!) { - return -1; - } - if (t1.name === t2.name) { - return 0; - } - return 1; - }); - - callback( - null, - _.map(terms, function (t, i) { - t.insertValue = t.insertValue || t.value; - t.value = '' + t.value; // normalize to strings - t.score = -i; - return t; - }) - ); + if (!context.asyncResultsState?.isLoading) { + const terms = getTerms(context, context.autoCompleteSet!); + const suggestions = getSuggestions(terms); + callback(null, suggestions); + } + + if (context.asyncResultsState) { + annotationControls.setAnnotation( + i18n.translate('console.autocomplete.fieldsFetchingAnnotation', { + defaultMessage: 'Fields fetching is in progress', + }) + ); + + context.asyncResultsState.results.then((r) => { + const asyncSuggestions = getSuggestions(getTerms(context, r)); + callback(null, asyncSuggestions); + annotationControls.removeAnnotation(); + }); + } } } catch (e) { // eslint-disable-next-line no-console @@ -1216,8 +1251,12 @@ export default function ({ _editSession: unknown, pos: Position, prefix: string, - callback: (e: Error | null, result: ResultTerm[] | null) => void - ) => getCompletions(pos, prefix, callback), + callback: (e: Error | null, result: ResultTerm[] | null) => void, + annotationControls: { + setAnnotation: (text: string) => void; + removeAnnotation: () => void; + } + ) => getCompletions(pos, prefix, callback, annotationControls), addReplacementInfoToContext, addChangeListener: () => editor.on('changeSelection', editorChangeListener), removeChangeListener: () => editor.off('changeSelection', editorChangeListener), diff --git a/src/plugins/console/public/lib/autocomplete/types.ts b/src/plugins/console/public/lib/autocomplete/types.ts index 15d32e6426a6c..a151c13f46c20 100644 --- a/src/plugins/console/public/lib/autocomplete/types.ts +++ b/src/plugins/console/public/lib/autocomplete/types.ts @@ -13,6 +13,7 @@ export interface ResultTerm { insertValue?: string; name?: string; value?: string; + score?: number; } export interface DataAutoCompleteRulesOneOf { @@ -25,6 +26,14 @@ export interface DataAutoCompleteRulesOneOf { export interface AutoCompleteContext { autoCompleteSet?: null | ResultTerm[]; + /** + * Stores a state for async results, e.g. fields suggestions based on the mappings definition. + */ + asyncResultsState?: { + isLoading: boolean; + lastFetched: number | null; + results: Promise; + }; endpoint?: null | { paramsAutocomplete: { getTopLevelComponents: (method?: string | null) => unknown; diff --git a/src/plugins/console/public/lib/autocomplete_entities/autocomplete_entities.test.js b/src/plugins/console/public/lib/autocomplete_entities/autocomplete_entities.test.js index 5349538799d9b..f856ef5750a17 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/autocomplete_entities.test.js +++ b/src/plugins/console/public/lib/autocomplete_entities/autocomplete_entities.test.js @@ -9,6 +9,9 @@ import '../../application/models/sense_editor/sense_editor.test.mocks'; import { setAutocompleteInfo, AutocompleteInfo } from '../../services'; import { expandAliases } from './expand_aliases'; +import { httpServiceMock } from '@kbn/core-http-browser-mocks'; +import { SettingsMock } from '../../services/settings.mock'; +import { StorageMock } from '../../services/storage.mock'; function fc(f1, f2) { if (f1.name < f2.name) { @@ -32,10 +35,20 @@ describe('Autocomplete entities', () => { let componentTemplate; let dataStream; let autocompleteInfo; + let settingsMock; + let httpMock; + beforeEach(() => { autocompleteInfo = new AutocompleteInfo(); setAutocompleteInfo(autocompleteInfo); mapping = autocompleteInfo.mapping; + + httpMock = httpServiceMock.createSetupContract(); + const storage = new StorageMock({}, 'test'); + settingsMock = new SettingsMock(storage); + + mapping.setup(httpMock, settingsMock); + alias = autocompleteInfo.alias; legacyTemplate = autocompleteInfo.legacyTemplate; indexTemplate = autocompleteInfo.indexTemplate; @@ -48,61 +61,98 @@ describe('Autocomplete entities', () => { }); describe('Mappings', function () { - test('Multi fields 1.0 style', function () { - mapping.loadMappings({ - index: { - properties: { - first_name: { - type: 'string', - index: 'analyzed', - path: 'just_name', - fields: { - any_name: { type: 'string', index: 'analyzed' }, - }, - }, - last_name: { - type: 'string', - index: 'no', - fields: { - raw: { type: 'string', index: 'analyzed' }, + describe('When fields autocomplete is disabled', () => { + beforeEach(() => { + settingsMock.getAutocomplete.mockReturnValue({ fields: false }); + }); + + test('does not return any suggestions', function () { + mapping.loadMappings({ + index: { + properties: { + first_name: { + type: 'string', + index: 'analyzed', + path: 'just_name', + fields: { + any_name: { type: 'string', index: 'analyzed' }, + }, }, }, }, - }, - }); + }); - expect(mapping.getMappings('index').sort(fc)).toEqual([ - f('any_name', 'string'), - f('first_name', 'string'), - f('last_name', 'string'), - f('last_name.raw', 'string'), - ]); + expect(mapping.getMappings('index').sort(fc)).toEqual([]); + }); }); - test('Simple fields', function () { - mapping.loadMappings({ - index: { - properties: { - str: { - type: 'string', - }, - number: { - type: 'int', + describe('When fields autocomplete is enabled', () => { + beforeEach(() => { + settingsMock.getAutocomplete.mockReturnValue({ fields: true }); + httpMock.get.mockReturnValue( + Promise.resolve({ + mappings: { index: { mappings: { properties: { '@timestamp': { type: 'date' } } } } }, + }) + ); + }); + + test('attempts to fetch mappings if not loaded', async () => { + const autoCompleteContext = {}; + let loadingIndicator; + + mapping.isLoading$.subscribe((v) => { + loadingIndicator = v; + }); + + // act + mapping.getMappings('index', [], autoCompleteContext); + + expect(autoCompleteContext.asyncResultsState.isLoading).toBe(true); + expect(loadingIndicator).toBe(true); + + expect(httpMock.get).toHaveBeenCalled(); + + const fields = await autoCompleteContext.asyncResultsState.results; + + expect(loadingIndicator).toBe(false); + expect(autoCompleteContext.asyncResultsState.isLoading).toBe(false); + expect(fields).toEqual([{ name: '@timestamp', type: 'date' }]); + }); + + test('Multi fields 1.0 style', function () { + mapping.loadMappings({ + index: { + properties: { + first_name: { + type: 'string', + index: 'analyzed', + path: 'just_name', + fields: { + any_name: { type: 'string', index: 'analyzed' }, + }, + }, + last_name: { + type: 'string', + index: 'no', + fields: { + raw: { type: 'string', index: 'analyzed' }, + }, + }, }, }, - }, - }); + }); - expect(mapping.getMappings('index').sort(fc)).toEqual([ - f('number', 'int'), - f('str', 'string'), - ]); - }); + expect(mapping.getMappings('index').sort(fc)).toEqual([ + f('any_name', 'string'), + f('first_name', 'string'), + f('last_name', 'string'), + f('last_name.raw', 'string'), + ]); + }); - test('Simple fields - 1.0 style', function () { - mapping.loadMappings({ - index: { - mappings: { + test('Simple fields', function () { + mapping.loadMappings({ + index: { properties: { str: { type: 'string', @@ -112,108 +162,130 @@ describe('Autocomplete entities', () => { }, }, }, - }, - }); + }); - expect(mapping.getMappings('index').sort(fc)).toEqual([ - f('number', 'int'), - f('str', 'string'), - ]); - }); + expect(mapping.getMappings('index').sort(fc)).toEqual([ + f('number', 'int'), + f('str', 'string'), + ]); + }); - test('Nested fields', function () { - mapping.loadMappings({ - index: { - properties: { - person: { - type: 'object', + test('Simple fields - 1.0 style', function () { + mapping.loadMappings({ + index: { + mappings: { properties: { - name: { - properties: { - first_name: { type: 'string' }, - last_name: { type: 'string' }, - }, + str: { + type: 'string', + }, + number: { + type: 'int', }, - sid: { type: 'string', index: 'not_analyzed' }, }, }, - message: { type: 'string' }, }, - }, - }); + }); - expect(mapping.getMappings('index', []).sort(fc)).toEqual([ - f('message'), - f('person.name.first_name'), - f('person.name.last_name'), - f('person.sid'), - ]); - }); + expect(mapping.getMappings('index').sort(fc)).toEqual([ + f('number', 'int'), + f('str', 'string'), + ]); + }); - test('Enabled fields', function () { - mapping.loadMappings({ - index: { - properties: { - person: { - type: 'object', - properties: { - name: { - type: 'object', - enabled: false, + test('Nested fields', function () { + mapping.loadMappings({ + index: { + properties: { + person: { + type: 'object', + properties: { + name: { + properties: { + first_name: { type: 'string' }, + last_name: { type: 'string' }, + }, + }, + sid: { type: 'string', index: 'not_analyzed' }, }, - sid: { type: 'string', index: 'not_analyzed' }, }, + message: { type: 'string' }, }, - message: { type: 'string' }, }, - }, - }); + }); - expect(mapping.getMappings('index', []).sort(fc)).toEqual([f('message'), f('person.sid')]); - }); + expect(mapping.getMappings('index', []).sort(fc)).toEqual([ + f('message'), + f('person.name.first_name'), + f('person.name.last_name'), + f('person.sid'), + ]); + }); - test('Path tests', function () { - mapping.loadMappings({ - index: { - properties: { - name1: { - type: 'object', - path: 'just_name', - properties: { - first1: { type: 'string' }, - last1: { type: 'string', index_name: 'i_last_1' }, + test('Enabled fields', function () { + mapping.loadMappings({ + index: { + properties: { + person: { + type: 'object', + properties: { + name: { + type: 'object', + enabled: false, + }, + sid: { type: 'string', index: 'not_analyzed' }, + }, }, + message: { type: 'string' }, }, - name2: { - type: 'object', - path: 'full', - properties: { - first2: { type: 'string' }, - last2: { type: 'string', index_name: 'i_last_2' }, + }, + }); + + expect(mapping.getMappings('index', []).sort(fc)).toEqual([f('message'), f('person.sid')]); + }); + + test('Path tests', function () { + mapping.loadMappings({ + index: { + properties: { + name1: { + type: 'object', + path: 'just_name', + properties: { + first1: { type: 'string' }, + last1: { type: 'string', index_name: 'i_last_1' }, + }, + }, + name2: { + type: 'object', + path: 'full', + properties: { + first2: { type: 'string' }, + last2: { type: 'string', index_name: 'i_last_2' }, + }, }, }, }, - }, - }); + }); - expect(mapping.getMappings().sort(fc)).toEqual([ - f('first1'), - f('i_last_1'), - f('name2.first2'), - f('name2.i_last_2'), - ]); - }); + expect(mapping.getMappings().sort(fc)).toEqual([ + f('first1'), + f('i_last_1'), + f('name2.first2'), + f('name2.i_last_2'), + ]); + }); - test('Use index_name tests', function () { - mapping.loadMappings({ - index: { - properties: { - last1: { type: 'string', index_name: 'i_last_1' }, + test('Use index_name tests', function () { + mapping.loadMappings({ + index: { + properties: { + last1: { type: 'string', index_name: 'i_last_1' }, + }, }, - }, - }); + }); - expect(mapping.getMappings().sort(fc)).toEqual([f('i_last_1')]); + expect(mapping.getMappings().sort(fc)).toEqual([f('i_last_1')]); + }); }); }); diff --git a/src/plugins/console/public/lib/autocomplete_entities/mapping.ts b/src/plugins/console/public/lib/autocomplete_entities/mapping.ts index 71e72dac0a280..3c383ca124167 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/mapping.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/mapping.ts @@ -7,9 +7,15 @@ */ import _ from 'lodash'; +import { BehaviorSubject } from 'rxjs'; import type { IndicesGetMappingResponse } from '@elastic/elasticsearch/lib/api/types'; +import { HttpSetup } from '@kbn/core-http-browser'; +import { type Settings } from '../../services'; +import { API_BASE_PATH } from '../../../common/constants'; +import type { ResultTerm, AutoCompleteContext } from '../autocomplete/types'; import { expandAliases } from './expand_aliases'; import type { Field, FieldMapping } from './types'; +import { type AutoCompleteEntitiesApiResponse } from './types'; function getFieldNamesFromProperties(properties: Record = {}) { const fieldList = Object.entries(properties).flatMap(([fieldName, fieldMapping]) => { @@ -70,22 +76,127 @@ function getFieldNamesFromFieldMapping( export interface BaseMapping { perIndexTypes: Record; - getMappings(indices: string | string[], types?: string | string[]): Field[]; + /** + * Fetches mappings definition + */ + fetchMappings(index: string): Promise; + + /** + * Retrieves mappings definition from cache, fetches if necessary. + */ + getMappings( + indices: string | string[], + types?: string | string[], + autoCompleteContext?: AutoCompleteContext + ): Field[]; + + /** + * Stores mappings definition + * @param mappings + */ loadMappings(mappings: IndicesGetMappingResponse): void; clearMappings(): void; } export class Mapping implements BaseMapping { + private http!: HttpSetup; + + private settings!: Settings; + + /** + * Map of the mappings of actual ES indices. + */ public perIndexTypes: Record = {}; - getMappings = (indices: string | string[], types?: string | string[]) => { + private readonly _isLoading$ = new BehaviorSubject(false); + + /** + * Indicates if mapping fetching is in progress. + */ + public readonly isLoading$ = this._isLoading$.asObservable(); + + /** + * Map of the currently loading mappings for index patterns specified by a user. + * @private + */ + private loadingState: Record = {}; + + public setup(http: HttpSetup, settings: Settings) { + this.http = http; + this.settings = settings; + } + + /** + * Fetches mappings of the requested indices. + * @param index + */ + async fetchMappings(index: string): Promise { + const response = await this.http.get( + `${API_BASE_PATH}/autocomplete_entities`, + { + query: { fields: true, fieldsIndices: index }, + } + ); + + return response.mappings; + } + + getMappings = ( + indices: string | string[], + types?: string | string[], + autoCompleteContext?: AutoCompleteContext + ) => { // get fields for indices and types. Both can be a list, a string or null (meaning all). let ret: Field[] = []; + + if (!this.settings.getAutocomplete().fields) return ret; + indices = expandAliases(indices); if (typeof indices === 'string') { const typeDict = this.perIndexTypes[indices] as Record; - if (!typeDict) { + + if (!typeDict || Object.keys(typeDict).length === 0) { + if (!autoCompleteContext) return ret; + + // Mappings fetching for the index is already in progress + if (this.loadingState[indices]) return ret; + + this.loadingState[indices] = true; + + if (!autoCompleteContext.asyncResultsState) { + autoCompleteContext.asyncResultsState = {} as AutoCompleteContext['asyncResultsState']; + } + + autoCompleteContext.asyncResultsState!.isLoading = true; + + autoCompleteContext.asyncResultsState!.results = new Promise( + (resolve, reject) => { + this._isLoading$.next(true); + + this.fetchMappings(indices as string) + .then((mapping) => { + this._isLoading$.next(false); + + autoCompleteContext.asyncResultsState!.isLoading = false; + autoCompleteContext.asyncResultsState!.lastFetched = Date.now(); + + // cache mappings + this.loadMappings(mapping); + + const mappings = this.getMappings(indices, types, autoCompleteContext); + delete this.loadingState[indices as string]; + resolve(mappings); + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + this._isLoading$.next(false); + delete this.loadingState[indices as string]; + }); + } + ); + return []; } @@ -108,7 +219,7 @@ export class Mapping implements BaseMapping { // multi index mode. Object.keys(this.perIndexTypes).forEach((index) => { if (!indices || indices.length === 0 || indices.includes(index)) { - ret.push(this.getMappings(index, types) as unknown as Field); + ret.push(this.getMappings(index, types, autoCompleteContext) as unknown as Field); } }); @@ -121,8 +232,6 @@ export class Mapping implements BaseMapping { }; loadMappings = (mappings: IndicesGetMappingResponse) => { - this.perIndexTypes = {}; - Object.entries(mappings).forEach(([index, indexMapping]) => { const normalizedIndexMappings: Record = {}; let transformedMapping: Record = indexMapping; diff --git a/src/plugins/console/public/services/autocomplete.ts b/src/plugins/console/public/services/autocomplete.ts index 3e1a38a514607..57324049bab94 100644 --- a/src/plugins/console/public/services/autocomplete.ts +++ b/src/plugins/console/public/services/autocomplete.ts @@ -53,7 +53,11 @@ export class AutocompleteInfo { const collaborator = this.mapping; return () => this.alias.getIndices(includeAliases, collaborator); case ENTITIES.FIELDS: - return this.mapping.getMappings(context.indices, context.types); + return this.mapping.getMappings( + context.indices, + context.types, + Object.getPrototypeOf(context) + ); case ENTITIES.INDEX_TEMPLATES: return () => this.indexTemplate.getTemplates(); case ENTITIES.COMPONENT_TEMPLATES: @@ -93,7 +97,6 @@ export class AutocompleteInfo { } private load(data: AutoCompleteEntitiesApiResponse) { - this.mapping.loadMappings(data.mappings); const collaborator = this.mapping; this.alias.loadAliases(data.aliases, collaborator); this.indexTemplate.loadTemplates(data.indexTemplates); diff --git a/src/plugins/console/public/services/settings.ts b/src/plugins/console/public/services/settings.ts index e4731dd3f3a31..4056d20063a3e 100644 --- a/src/plugins/console/public/services/settings.ts +++ b/src/plugins/console/public/services/settings.ts @@ -14,7 +14,12 @@ export const DEFAULT_SETTINGS = Object.freeze({ pollInterval: 60000, tripleQuotes: true, wrapMode: true, - autocomplete: Object.freeze({ fields: true, indices: true, templates: true, dataStreams: true }), + autocomplete: Object.freeze({ + fields: true, + indices: true, + templates: true, + dataStreams: true, + }), isHistoryEnabled: true, isKeyboardShortcutsEnabled: true, }); diff --git a/src/plugins/console/public/types/core_editor.ts b/src/plugins/console/public/types/core_editor.ts index 1c9d6352914a2..34f44578a0bce 100644 --- a/src/plugins/console/public/types/core_editor.ts +++ b/src/plugins/console/public/types/core_editor.ts @@ -7,6 +7,7 @@ */ import type { Editor } from 'brace'; +import { ResultTerm } from '../lib/autocomplete/types'; import { TokensProvider } from './tokens_provider'; import { Token } from './token'; @@ -23,7 +24,11 @@ export type EditorEvent = export type AutoCompleterFunction = ( pos: Position, prefix: string, - callback: (...args: unknown[]) => void + callback: (e: Error | null, result: ResultTerm[] | null) => void, + annotationControls: { + setAnnotation: (text: string) => void; + removeAnnotation: () => void; + } ) => void; export interface Position { diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts index 8bd5f8ee50b48..2d19de0a56e74 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/index.ts @@ -6,26 +6,25 @@ * Side Public License, v 1. */ -import { parse } from 'query-string'; import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import type { RouteDependencies } from '../../..'; -interface SettingsToRetrieve { - indices: boolean; - fields: boolean; - templates: boolean; - dataStreams: boolean; -} +import { autoCompleteEntitiesValidationConfig, type SettingsToRetrieve } from './validation_config'; const MAX_RESPONSE_SIZE = 10 * 1024 * 1024; // 10MB // Limit the response size to 10MB, because the response can be very large and sending it to the client // can cause the browser to hang. const getMappings = async (settings: SettingsToRetrieve, esClient: IScopedClusterClient) => { - if (settings.fields) { - const mappings = await esClient.asInternalUser.indices.getMapping(undefined, { - maxResponseSize: MAX_RESPONSE_SIZE, - maxCompressedResponseSize: MAX_RESPONSE_SIZE, - }); + if (settings.fields && settings.fieldsIndices) { + const mappings = await esClient.asInternalUser.indices.getMapping( + { + index: settings.fieldsIndices, + }, + { + maxResponseSize: MAX_RESPONSE_SIZE, + maxCompressedResponseSize: MAX_RESPONSE_SIZE, + } + ); return mappings; } // If the user doesn't want autocomplete suggestions, then clear any that exist. @@ -87,20 +86,11 @@ export const registerAutocompleteEntitiesRoute = (deps: RouteDependencies) => { options: { tags: ['access:console'], }, - validate: false, + validate: autoCompleteEntitiesValidationConfig, }, async (context, request, response) => { const esClient = (await context.core).elasticsearch.client; - const settings = parse(request.url.search, { - parseBooleans: true, - }) as unknown as SettingsToRetrieve; - - // If no settings are specified, then return 400. - if (Object.keys(settings).length === 0) { - return response.badRequest({ - body: 'Request must contain at least one of the following parameters: indices, fields, templates, dataStreams', - }); - } + const settings = request.query; // Wait for all requests to complete, in case one of them fails return the successfull ones const results = await Promise.allSettled([ diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/validation_config.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/validation_config.ts new file mode 100644 index 0000000000000..48bab100d0b61 --- /dev/null +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/validation_config.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const autoCompleteEntitiesValidationConfig = { + query: schema.object( + { + indices: schema.maybe(schema.boolean()), + fields: schema.maybe(schema.boolean()), + templates: schema.maybe(schema.boolean()), + dataStreams: schema.maybe(schema.boolean()), + /** + * Comma separated list of indices for mappings retrieval. + */ + fieldsIndices: schema.maybe(schema.string()), + }, + { + validate: (payload) => { + if (Object.keys(payload).length === 0) { + return 'The request must contain at least one of the following parameters: indices, fields, templates, dataStreams.'; + } + }, + } + ), +}; + +export type SettingsToRetrieve = TypeOf; diff --git a/src/plugins/console/tsconfig.json b/src/plugins/console/tsconfig.json index 0dcc23b1c060c..43b94e47eedd4 100644 --- a/src/plugins/console/tsconfig.json +++ b/src/plugins/console/tsconfig.json @@ -30,6 +30,7 @@ "@kbn/core-http-router-server-internal", "@kbn/web-worker-stub", "@kbn/core-elasticsearch-server", + "@kbn/core-http-browser-mocks", ], "exclude": [ "target/**/*", diff --git a/test/api_integration/apis/console/autocomplete_entities.ts b/test/api_integration/apis/console/autocomplete_entities.ts index 6bd899c979a2b..6e13b2fb2856b 100644 --- a/test/api_integration/apis/console/autocomplete_entities.ts +++ b/test/api_integration/apis/console/autocomplete_entities.ts @@ -128,7 +128,7 @@ export default ({ getService }: FtrProviderContext) => { return await supertest.get('/api/console/autocomplete_entities').query(query); }; - describe('/api/console/autocomplete_entities', () => { + describe('/api/console/autocomplete_entities', function () { const indexName = 'test-index-1'; const aliasName = 'test-alias-1'; const indexTemplateName = 'test-index-template-1'; @@ -238,9 +238,17 @@ export default ({ getService }: FtrProviderContext) => { expect(body.mappings).to.eql({}); }); - it('should return mappings with fields setting is set to true', async () => { + it('should not return mappings with fields setting is set to true without the list of indices is provided', async () => { const response = await sendRequest({ fields: true }); + const { body, status } = response; + expect(status).to.be(200); + expect(Object.keys(body.mappings)).to.not.contain(indexName); + }); + + it('should return mappings with fields setting is set to true and the list of indices is provided', async () => { + const response = await sendRequest({ fields: true, fieldsIndices: indexName }); + const { body, status } = response; expect(status).to.be(200); expect(Object.keys(body.mappings)).to.contain(indexName); From 9a3e3197481c37db6209f0f0fd00f570d1013cc4 Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Mon, 6 Feb 2023 14:31:49 -0600 Subject: [PATCH 013/134] [Enterprise Search][Engines] Clean-up work (#150232) ## Summary Clean-up work for engines prior to 8.7 feature freeze. Note engines will be feature flagged in this release. The UI will only be reachable when the feature flag is turned on by enabling a flag in the enterprise search yaml config. - Adds telemetry tracking to all user actions - Removes the Documents side nav item, this page has been removed from the scope of MVP - Added an engines doc link, this is still TBD path may still be updated to a landing page for 8.7 --- packages/kbn-doc-links/src/get_doc_links.ts | 1 + packages/kbn-doc-links/src/types.ts | 1 + .../components/engine/add_indices_flyout.tsx | 13 ++++- .../components/engine/engine_indices.tsx | 15 +++++- .../engine/engine_view_header_actions.tsx | 7 +++ .../components/engine/header_docs_action.tsx | 3 +- .../components/tables/engines_table.tsx | 9 +++- .../engines/create_engine_flyout.tsx | 4 +- .../engines/delete_engine_modal.tsx | 6 +++ .../components/engines/engines_list.tsx | 5 +- .../enterprise_search_content/routes.ts | 1 - .../shared/doc_links/doc_links.ts | 3 ++ .../applications/shared/layout/nav.test.tsx | 25 ++++----- .../public/applications/shared/layout/nav.tsx | 52 ++++++++----------- 14 files changed, 88 insertions(+), 57 deletions(-) diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index 0b253cedbc672..6b30af3f39097 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -135,6 +135,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { crawlerOverview: `${ENTERPRISE_SEARCH_DOCS}crawler.html`, deployTrainedModels: `${MACHINE_LEARNING_DOCS}ml-nlp-deploy-models.html`, documentLevelSecurity: `${ELASTICSEARCH_DOCS}document-level-security.html`, + engines: `${ENTERPRISE_SEARCH_DOCS}engines.html`, ingestPipelines: `${ENTERPRISE_SEARCH_DOCS}ingest-pipelines.html`, languageAnalyzers: `${ELASTICSEARCH_DOCS}analysis-lang-analyzer.html`, languageClients: `${ENTERPRISE_SEARCH_DOCS}programming-language-clients.html`, diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts index dbfb37905172f..a6284078d3d8d 100644 --- a/packages/kbn-doc-links/src/types.ts +++ b/packages/kbn-doc-links/src/types.ts @@ -120,6 +120,7 @@ export interface DocLinks { readonly crawlerOverview: string; readonly deployTrainedModels: string; readonly documentLevelSecurity: string; + readonly engines: string; readonly ingestPipelines: string; readonly languageAnalyzers: string; readonly languageClients: string; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/add_indices_flyout.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/add_indices_flyout.tsx index 852e3698b968c..22ab72ba3a405 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/add_indices_flyout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/add_indices_flyout.tsx @@ -99,7 +99,12 @@ export const AddIndicesFlyout: React.FC = ({ onClose }) = - + {i18n.translate( 'xpack.enterpriseSearch.content.engine.indices.addIndicesFlyout.submitButton', { defaultMessage: 'Add selected' } @@ -107,7 +112,11 @@ export const AddIndicesFlyout: React.FC = ({ onClose }) = - + {i18n.translate( 'xpack.enterpriseSearch.content.engine.indices.addIndicesFlyout.cancelButton', { defaultMessage: 'Cancel' } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_indices.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_indices.tsx index 0bcdde5e47647..da664b3d97490 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_indices.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_indices.tsx @@ -28,6 +28,7 @@ import { indexHealthToHealthColor } from '../../../shared/constants/health_color import { generateEncodedPath } from '../../../shared/encode_path_params'; import { KibanaLogic } from '../../../shared/kibana'; import { EuiLinkTo } from '../../../shared/react_router_helpers'; +import { TelemetryLogic } from '../../../shared/telemetry/telemetry_logic'; import { SEARCH_INDEX_PATH, EngineViewTabs } from '../../routes'; import { IngestionMethod } from '../../types'; @@ -40,6 +41,7 @@ import { EngineIndicesLogic } from './engine_indices_logic'; import { EngineViewHeaderActions } from './engine_view_header_actions'; export const EngineIndices: React.FC = () => { + const { sendEnterpriseSearchTelemetry } = useActions(TelemetryLogic); const { engineData, engineName, isLoadingEngine, addIndicesFlyoutOpen } = useValues(EngineIndicesLogic); const { removeIndexFromEngine, openAddIndicesFlyout, closeAddIndicesFlyout } = @@ -157,7 +159,13 @@ export const EngineIndices: React.FC = () => { }, } ), - onClick: (index) => setConfirmRemoveIndex(index.name), + onClick: (index) => { + setConfirmRemoveIndex(index.name); + sendEnterpriseSearchTelemetry({ + action: 'clicked', + metric: 'entSearchContent-engines-indices-removeIndex', + }); + }, type: 'icon', }, ], @@ -180,6 +188,7 @@ export const EngineIndices: React.FC = () => { { onConfirm={() => { removeIndexFromEngine(removeIndexConfirm); setConfirmRemoveIndex(null); + sendEnterpriseSearchTelemetry({ + action: 'clicked', + metric: 'entSearchContent-engines-indices-removeIndexConfirm', + }); }} title={i18n.translate( 'xpack.enterpriseSearch.content.engine.indices.removeIndexConfirm.title', diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_header_actions.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_header_actions.tsx index ba563a9c23dbc..bd48ee8d6d294 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_header_actions.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_header_actions.tsx @@ -13,12 +13,15 @@ import { EuiPopover, EuiButtonIcon, EuiText, EuiContextMenu, EuiIcon } from '@el import { i18n } from '@kbn/i18n'; +import { TelemetryLogic } from '../../../shared/telemetry/telemetry_logic'; + import { EngineViewLogic } from './engine_view_logic'; export const EngineViewHeaderActions: React.FC = () => { const { engineData } = useValues(EngineViewLogic); const { openDeleteEngineModal } = useActions(EngineViewLogic); + const { sendEnterpriseSearchTelemetry } = useActions(TelemetryLogic); const [isActionsPopoverOpen, setIsActionsPopoverOpen] = useState(false); const toggleActionsPopover = () => setIsActionsPopoverOpen((isPopoverOpen) => !isPopoverOpen); @@ -66,6 +69,10 @@ export const EngineViewHeaderActions: React.FC = () => { onClick: () => { if (engineData) { openDeleteEngineModal(); + sendEnterpriseSearchTelemetry({ + action: 'clicked', + metric: 'entSearchContent-engines-engineView-deleteEngine', + }); } }, size: 's', diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/header_docs_action.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/header_docs_action.tsx index 93544cc2e9bbd..e20fbc81689a4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/header_docs_action.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/header_docs_action.tsx @@ -15,8 +15,9 @@ export const EngineHeaderDocsAction: React.FC = () => ( diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/components/tables/engines_table.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/components/tables/engines_table.tsx index b3c5c46b38128..017564343f11e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/components/tables/engines_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/components/tables/engines_table.tsx @@ -7,7 +7,7 @@ import React from 'react'; -import { useValues } from 'kea'; +import { useValues, useActions } from 'kea'; import { CriteriaWithPagination, @@ -29,6 +29,7 @@ import { FormattedDateTime } from '../../../../../shared/formatted_date_time'; import { KibanaLogic } from '../../../../../shared/kibana'; import { pageToPagination } from '../../../../../shared/pagination/page_to_pagination'; import { EuiLinkTo } from '../../../../../shared/react_router_helpers'; +import { TelemetryLogic } from '../../../../../shared/telemetry/telemetry_logic'; import { ENGINE_PATH } from '../../../../routes'; @@ -50,6 +51,7 @@ export const EnginesListTable: React.FC = ({ viewEngineIndices, }) => { const { navigateToUrl } = useValues(KibanaLogic); + const { sendEnterpriseSearchTelemetry } = useActions(TelemetryLogic); const columns: Array> = [ { field: 'name', @@ -93,6 +95,7 @@ export const EnginesListTable: React.FC = ({ size="s" className="engineListTableFlyoutButton" data-test-subj="engineListTableIndicesFlyoutButton" + data-telemetry-id="entSearchContent-engines-table-viewEngineIndices" onClick={() => viewEngineIndices(engine.name)} > = ({ ), onClick: (engine) => { onDelete(engine); + sendEnterpriseSearchTelemetry({ + action: 'clicked', + metric: 'entSearchContent-engines-table-deleteEngine', + }); }, }, ], diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/create_engine_flyout.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/create_engine_flyout.tsx index 06a2509edad53..ae7f7423be7ce 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/create_engine_flyout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/create_engine_flyout.tsx @@ -36,7 +36,7 @@ import { ElasticsearchIndexWithIngestion } from '../../../../../common/types/ind import { isNotNullish } from '../../../../../common/utils/is_not_nullish'; import { CANCEL_BUTTON_LABEL } from '../../../shared/constants'; - +import { docLinks } from '../../../shared/doc_links'; import { getErrorsFromHttpResponse } from '../../../shared/flash_messages/handle_api_errors'; import { indexToOption, IndicesSelectComboBox } from './components/indices_select_combobox'; @@ -85,7 +85,7 @@ export const CreateEngineFlyout = ({ onClose }: CreateEngineFlyoutProps) => { values={{ enginesDocsLink: ( = ({ engineName, onClose }) => { const { deleteEngine } = useActions(EnginesListLogic); + const { sendEnterpriseSearchTelemetry } = useActions(TelemetryLogic); const { isDeleteLoading } = useValues(EnginesListLogic); return ( = ({ engineName onCancel={onClose} onConfirm={() => { deleteEngine({ engineName }); + sendEnterpriseSearchTelemetry({ + action: 'clicked', + metric: 'entSearchContent-engines-engineView-deleteEngineConfirm', + }); }} cancelButtonText={CANCEL_BUTTON_LABEL} confirmButtonText={i18n.translate( diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_list.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_list.tsx index 76e343e234ff3..b927e499ff06c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_list.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engines/engines_list.tsx @@ -16,6 +16,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage, FormattedNumber } from '@kbn/i18n-react'; import { INPUT_THROTTLE_DELAY_MS } from '../../../shared/constants/timers'; +import { docLinks } from '../../../shared/doc_links'; import { EnterpriseSearchEnginesPageTemplate } from '../layout/engines_page_template'; @@ -102,12 +103,10 @@ export const EnginesList: React.FC = () => { documentationUrl: ( - {' '} - {/* TODO: navigate to documentation url */}{' '} {i18n.translate('xpack.enterpriseSearch.content.engines.documentation', { defaultMessage: 'explore our Engines documentation', })} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/routes.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/routes.ts index 4185addb31445..57ff05050ebae 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/routes.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/routes.ts @@ -32,7 +32,6 @@ export const ENGINE_TAB_PATH = `${ENGINE_PATH}/:tabId`; export enum EngineViewTabs { OVERVIEW = 'overview', INDICES = 'indices', - DOCUMENTS = 'documents', SCHEMA = 'schema', PREVIEW = 'preview', API = 'api', diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/doc_links/doc_links.ts b/x-pack/plugins/enterprise_search/public/applications/shared/doc_links/doc_links.ts index cc1356fc0c140..073ffe6abca3f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/doc_links/doc_links.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/doc_links/doc_links.ts @@ -71,6 +71,7 @@ class DocLinks { public elasticsearchMapping: string; public elasticsearchSecureCluster: string; public enterpriseSearchConfig: string; + public enterpriseSearchEngines: string; public enterpriseSearchMailService: string; public enterpriseSearchTroubleshootSetup: string; public enterpriseSearchUsersAccess: string; @@ -186,6 +187,7 @@ class DocLinks { this.elasticsearchMapping = ''; this.elasticsearchSecureCluster = ''; this.enterpriseSearchConfig = ''; + this.enterpriseSearchEngines = ''; this.enterpriseSearchMailService = ''; this.enterpriseSearchTroubleshootSetup = ''; this.enterpriseSearchUsersAccess = ''; @@ -302,6 +304,7 @@ class DocLinks { this.elasticsearchMapping = docLinks.links.elasticsearch.mapping; this.elasticsearchSecureCluster = docLinks.links.elasticsearch.secureCluster; this.enterpriseSearchConfig = docLinks.links.enterpriseSearch.configuration; + this.enterpriseSearchEngines = docLinks.links.enterpriseSearch.engines; this.enterpriseSearchMailService = docLinks.links.enterpriseSearch.mailService; this.enterpriseSearchTroubleshootSetup = docLinks.links.enterpriseSearch.troubleshootSetup; this.enterpriseSearchUsersAccess = docLinks.links.enterpriseSearch.usersAccess; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx index 17ce1fce0512f..fac9d911be6c8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.test.tsx @@ -468,21 +468,16 @@ describe('useEnterpriseSearchEngineNav', () => { id: 'enterpriseSearchEngineIndices', name: 'Indices', }, - { - href: `/app/enterprise_search/content/engines/${engineName}/documents`, - id: 'enterpriseSearchEngineDocuments', - name: 'Documents', - }, - { - href: `/app/enterprise_search/content/engines/${engineName}/schema`, - id: 'enterpriseSearchEngineSchema', - name: 'Schema', - }, - { - href: `/app/enterprise_search/content/engines/${engineName}/preview`, - id: 'enterpriseSearchEnginePreview', - name: 'Preview', - }, + // { + // href: `/app/enterprise_search/content/engines/${engineName}/schema`, + // id: 'enterpriseSearchEngineSchema', + // name: 'Schema', + // }, + // { + // href: `/app/enterprise_search/content/engines/${engineName}/preview`, + // id: 'enterpriseSearchEnginePreview', + // name: 'Preview', + // }, { href: `/app/enterprise_search/content/engines/${engineName}/api`, id: 'enterpriseSearchEngineAPI', diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx index a6e550e7c58b2..3d81f5d9d7c34 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/layout/nav.tsx @@ -271,37 +271,27 @@ export const useEnterpriseSearchEngineNav = (engineName?: string, isEmptyState?: to: `${enginePath}/${EngineViewTabs.INDICES}`, }), }, - - { - id: 'enterpriseSearchEngineDocuments', - name: i18n.translate('xpack.enterpriseSearch.nav.engine.documentsTitle', { - defaultMessage: 'Documents', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: `${enginePath}/${EngineViewTabs.DOCUMENTS}`, - }), - }, - { - id: 'enterpriseSearchEngineSchema', - name: i18n.translate('xpack.enterpriseSearch.nav.engine.schemaTitle', { - defaultMessage: 'Schema', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: `${enginePath}/${EngineViewTabs.SCHEMA}`, - }), - }, - { - id: 'enterpriseSearchEnginePreview', - name: i18n.translate('xpack.enterpriseSearch.nav.engine.previewTitle', { - defaultMessage: 'Preview', - }), - ...generateNavLink({ - shouldNotCreateHref: true, - to: `${enginePath}/${EngineViewTabs.PREVIEW}`, - }), - }, + // { + // id: 'enterpriseSearchEngineSchema', + // name: i18n.translate('xpack.enterpriseSearch.nav.engine.schemaTitle', { + // defaultMessage: 'Schema', + // }), + // ...generateNavLink({ + // shouldNotCreateHref: true, + // to: `${enginePath}/${EngineViewTabs.SCHEMA}`, + // }), + // }, + // Hidden until Preview page is available + // { + // id: 'enterpriseSearchEnginePreview', + // name: i18n.translate('xpack.enterpriseSearch.nav.engine.previewTitle', { + // defaultMessage: 'Preview', + // }), + // ...generateNavLink({ + // shouldNotCreateHref: true, + // to: `${enginePath}/${EngineViewTabs.PREVIEW}`, + // }), + // }, { id: 'enterpriseSearchEngineAPI', name: i18n.translate('xpack.enterpriseSearch.nav.engine.apiTitle', { From c3ea5e5b3a2f0e13e743eea059404c81a07576c1 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:38:36 -0500 Subject: [PATCH 014/134] [Cases] Improve functional tests (#150117) This PR tries to improve how often our functional tests succeeds. I also tried cleaning up a few things that seemed to be slowing the tests down and also causing errors when the tests were run individually. Fixes: https://github.com/elastic/kibana/issues/145271 Notable changes: - I added a value to the `property-actions*` in most places so that the functional tests can distinguish between a description, comment, or the case ellipses this seems to work consistently where other methods have not Flaky test run: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/1871 --- .../case_action_bar/actions.test.tsx | 39 +++-- .../components/case_action_bar/actions.tsx | 2 +- .../components/case_action_bar/index.test.tsx | 16 +- .../components/property_actions/index.tsx | 143 ++++++++++-------- .../user_actions/comment/comment.test.tsx | 24 +-- .../user_actions/description.test.tsx | 16 +- .../components/user_actions/index.test.tsx | 36 +++-- .../alert_property_actions.test.tsx | 36 ++--- .../description_property_actions.test.tsx | 28 ++-- .../description_property_actions.tsx | 8 +- .../property_actions.test.tsx | 12 +- .../property_actions/property_actions.tsx | 12 +- ...ered_attachments_property_actions.test.tsx | 28 ++-- .../user_comment_property_actions.test.tsx | 40 ++--- x-pack/test/functional/services/cases/list.ts | 3 - .../services/cases/single_case_view.ts | 12 +- .../apps/cases/deletion.ts | 6 +- .../apps/cases/view_case.ts | 28 +--- 18 files changed, 263 insertions(+), 226 deletions(-) diff --git a/x-pack/plugins/cases/public/components/case_action_bar/actions.test.tsx b/x-pack/plugins/cases/public/components/case_action_bar/actions.test.tsx index 195d02f7931cf..e04070fecf9ff 100644 --- a/x-pack/plugins/cases/public/components/case_action_bar/actions.test.tsx +++ b/x-pack/plugins/cases/public/components/case_action_bar/actions.test.tsx @@ -46,8 +46,11 @@ describe('CaseView actions', () => { ); expect(wrapper.find('[data-test-subj="confirm-delete-case-modal"]').exists()).toBeFalsy(); - wrapper.find('button[data-test-subj="property-actions-ellipses"]').first().simulate('click'); - wrapper.find('button[data-test-subj="property-actions-trash"]').simulate('click'); + wrapper + .find('button[data-test-subj="property-actions-case-ellipses"]') + .first() + .simulate('click'); + wrapper.find('button[data-test-subj="property-actions-case-trash"]').simulate('click'); expect(wrapper.find('[data-test-subj="confirm-delete-case-modal"]').exists()).toBeTruthy(); }); @@ -67,8 +70,11 @@ describe('CaseView actions', () => { ); - wrapper.find('button[data-test-subj="property-actions-ellipses"]').first().simulate('click'); - wrapper.find('button[data-test-subj="property-actions-copyClipboard"]').simulate('click'); + wrapper + .find('button[data-test-subj="property-actions-case-ellipses"]') + .first() + .simulate('click'); + wrapper.find('button[data-test-subj="property-actions-case-copyClipboard"]').simulate('click'); expect(navigator.clipboard.writeText).toHaveBeenCalledWith(basicCase.id); @@ -85,9 +91,14 @@ describe('CaseView actions', () => { ); expect(wrapper.find('[data-test-subj="confirm-delete-case-modal"]').exists()).toBeFalsy(); - wrapper.find('button[data-test-subj="property-actions-ellipses"]').first().simulate('click'); - expect(wrapper.find('[data-test-subj="property-actions-trash"]').exists()).toBeFalsy(); - expect(wrapper.find('[data-test-subj="property-actions-copyClipboard"]').exists()).toBeTruthy(); + wrapper + .find('button[data-test-subj="property-actions-case-ellipses"]') + .first() + .simulate('click'); + expect(wrapper.find('[data-test-subj="property-actions-case-trash"]').exists()).toBeFalsy(); + expect( + wrapper.find('[data-test-subj="property-actions-case-copyClipboard"]').exists() + ).toBeTruthy(); }); it('toggle delete modal and confirm', async () => { @@ -101,8 +112,11 @@ describe('CaseView actions', () => { ); - wrapper.find('button[data-test-subj="property-actions-ellipses"]').first().simulate('click'); - wrapper.find('button[data-test-subj="property-actions-trash"]').simulate('click'); + wrapper + .find('button[data-test-subj="property-actions-case-ellipses"]') + .first() + .simulate('click'); + wrapper.find('button[data-test-subj="property-actions-case-trash"]').simulate('click'); expect(wrapper.find('[data-test-subj="confirm-delete-case-modal"]').exists()).toBeTruthy(); wrapper.find('button[data-test-subj="confirmModalConfirmButton"]').simulate('click'); @@ -126,9 +140,12 @@ describe('CaseView actions', () => { expect(wrapper.find('[data-test-subj="confirm-delete-case-modal"]').exists()).toBeFalsy(); - wrapper.find('button[data-test-subj="property-actions-ellipses"]').first().simulate('click'); + wrapper + .find('button[data-test-subj="property-actions-case-ellipses"]') + .first() + .simulate('click'); expect( - wrapper.find('[data-test-subj="property-actions-popout"]').first().prop('aria-label') + wrapper.find('[data-test-subj="property-actions-case-popout"]').first().prop('aria-label') ).toEqual(i18n.VIEW_INCIDENT(basicPush.externalTitle)); }); }); diff --git a/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx b/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx index d464946b60f82..87cd1fc732a30 100644 --- a/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx +++ b/x-pack/plugins/cases/public/components/case_action_bar/actions.tsx @@ -84,7 +84,7 @@ const ActionsComponent: React.FC = ({ caseData, currentExternal return ( - + {isModalVisible ? ( { ); - userEvent.click(screen.getByTestId('property-actions-ellipses')); + userEvent.click(screen.getByTestId('property-actions-case-ellipses')); expect(queryByText('Delete case')).not.toBeInTheDocument(); - expect(queryByTestId('property-actions-trash')).not.toBeInTheDocument(); - expect(queryByTestId('property-actions-copyClipboard')).toBeInTheDocument(); + expect(queryByTestId('property-actions-case-trash')).not.toBeInTheDocument(); + expect(queryByTestId('property-actions-case-copyClipboard')).toBeInTheDocument(); }); it('should show the the delete item in the menu when the user does have delete privileges', () => { @@ -220,7 +220,7 @@ describe('CaseActionBar', () => { ); - userEvent.click(screen.getByTestId('property-actions-ellipses')); + userEvent.click(screen.getByTestId('property-actions-case-ellipses')); expect(queryByText('Delete case')).toBeInTheDocument(); }); @@ -239,10 +239,10 @@ describe('CaseActionBar', () => { ); - userEvent.click(screen.getByTestId('property-actions-ellipses')); + userEvent.click(screen.getByTestId('property-actions-case-ellipses')); await waitFor(() => { - expect(screen.getByTestId('property-actions-popout')).toBeInTheDocument(); + expect(screen.getByTestId('property-actions-case-popout')).toBeInTheDocument(); }); }); @@ -253,8 +253,8 @@ describe('CaseActionBar', () => { ); - userEvent.click(screen.getByTestId('property-actions-ellipses')); + userEvent.click(screen.getByTestId('property-actions-case-ellipses')); - expect(screen.queryByTestId('property-actions-popout')).not.toBeInTheDocument(); + expect(screen.queryByTestId('property-actions-case-popout')).not.toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cases/public/components/property_actions/index.tsx b/x-pack/plugins/cases/public/components/property_actions/index.tsx index 61f4c4b6f2cfe..4de52d551bf2f 100644 --- a/x-pack/plugins/cases/public/components/property_actions/index.tsx +++ b/x-pack/plugins/cases/public/components/property_actions/index.tsx @@ -17,86 +17,101 @@ export interface PropertyActionButtonProps { iconType: string; label: string; color?: EuiButtonProps['color']; + customDataTestSubj?: string; } const ComponentId = 'property-actions'; const PropertyActionButton = React.memo( - ({ disabled = false, onClick, iconType, label, color }) => ( - - {label} - - ) + ({ disabled = false, onClick, iconType, label, color, customDataTestSubj }) => { + const dataTestSubjPrepend = makeDataTestSubjPrepend(customDataTestSubj); + + return ( + + {label} + + ); + } ); PropertyActionButton.displayName = 'PropertyActionButton'; export interface PropertyActionsProps { propertyActions: PropertyActionButtonProps[]; + customDataTestSubj?: string; } -export const PropertyActions = React.memo(({ propertyActions }) => { - const [showActions, setShowActions] = useState(false); +export const PropertyActions = React.memo( + ({ propertyActions, customDataTestSubj }) => { + const [showActions, setShowActions] = useState(false); - const onButtonClick = useCallback(() => { - setShowActions((prevShowActions) => !prevShowActions); - }, []); + const onButtonClick = useCallback(() => { + setShowActions((prevShowActions) => !prevShowActions); + }, []); - const onClosePopover = useCallback((cb?: () => void) => { - setShowActions(false); - if (cb != null) { - cb(); - } - }, []); - - return ( - + const onClosePopover = useCallback((cb?: () => void) => { + setShowActions(false); + if (cb != null) { + cb(); } - id="settingsPopover" - isOpen={showActions} - closePopover={onClosePopover} - repositionOnScroll - > - + } + id="settingsPopover" + isOpen={showActions} + closePopover={onClosePopover} + repositionOnScroll > - {propertyActions.map((action, key) => ( - - - onClosePopover(action.onClick)} - /> - - - ))} - - - ); -}); + + {propertyActions.map((action, key) => ( + + + onClosePopover(action.onClick)} + customDataTestSubj={customDataTestSubj} + /> + + + ))} + + + ); + } +); PropertyActions.displayName = 'PropertyActions'; + +const makeDataTestSubjPrepend = (customDataTestSubj?: string) => { + return customDataTestSubj == null ? ComponentId : `${ComponentId}-${customDataTestSubj}`; +}; diff --git a/x-pack/plugins/cases/public/components/user_actions/comment/comment.test.tsx b/x-pack/plugins/cases/public/components/user_actions/comment/comment.test.tsx index 77f1af32529e4..2ecc4b32d0837 100644 --- a/x-pack/plugins/cases/public/components/user_actions/comment/comment.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/comment/comment.test.tsx @@ -223,13 +223,13 @@ describe('createCommentUserActionBuilder', () => { ); expect(result.getByText('Solve this fast!')).toBeInTheDocument(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-pencil')); + expect(result.queryByTestId('property-actions-user-action-pencil')).toBeInTheDocument(); + userEvent.click(result.getByTestId('property-actions-user-action-pencil')); await waitFor(() => { expect(builderArgs.handleManageMarkdownEditId).toHaveBeenCalledWith('basic-comment-id'); @@ -254,13 +254,13 @@ describe('createCommentUserActionBuilder', () => { ); expect(result.getByText('Solve this fast!')).toBeInTheDocument(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-quote')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-quote')); + expect(result.queryByTestId('property-actions-user-action-quote')).toBeInTheDocument(); + userEvent.click(result.getByTestId('property-actions-user-action-quote')); await waitFor(() => { expect(builderArgs.handleManageQuote).toHaveBeenCalledWith('Solve this fast!'); @@ -769,14 +769,14 @@ describe('createCommentUserActionBuilder', () => { }); const deleteAttachment = async (result: RenderResult, deleteIcon: string, buttonLabel: string) => { - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId(`property-actions-${deleteIcon}`)).toBeInTheDocument(); + expect(result.queryByTestId(`property-actions-user-action-${deleteIcon}`)).toBeInTheDocument(); - userEvent.click(result.getByTestId(`property-actions-${deleteIcon}`)); + userEvent.click(result.getByTestId(`property-actions-user-action-${deleteIcon}`)); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); diff --git a/x-pack/plugins/cases/public/components/user_actions/description.test.tsx b/x-pack/plugins/cases/public/components/user_actions/description.test.tsx index 9fddcbb3cce6a..5b519b6739bdd 100644 --- a/x-pack/plugins/cases/public/components/user_actions/description.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/description.test.tsx @@ -58,13 +58,13 @@ describe('createDescriptionUserActionBuilder ', () => { ); - expect(res.getByTestId('property-actions')).toBeInTheDocument(); + expect(res.getByTestId('property-actions-description')).toBeInTheDocument(); - userEvent.click(res.getByTestId('property-actions-ellipses')); + userEvent.click(res.getByTestId('property-actions-description-ellipses')); await waitForEuiPopoverOpen(); - expect(res.queryByTestId('property-actions-pencil')).toBeInTheDocument(); - userEvent.click(res.getByTestId('property-actions-pencil')); + expect(res.queryByTestId('property-actions-description-pencil')).toBeInTheDocument(); + userEvent.click(res.getByTestId('property-actions-description-pencil')); await waitFor(() => { expect(builderArgs.handleManageMarkdownEditId).toHaveBeenCalledWith('description'); @@ -84,13 +84,13 @@ describe('createDescriptionUserActionBuilder ', () => { ); - expect(res.getByTestId('property-actions')).toBeInTheDocument(); + expect(res.getByTestId('property-actions-description')).toBeInTheDocument(); - userEvent.click(res.getByTestId('property-actions-ellipses')); + userEvent.click(res.getByTestId('property-actions-description-ellipses')); await waitForEuiPopoverOpen(); - expect(res.queryByTestId('property-actions-quote')).toBeInTheDocument(); - userEvent.click(res.getByTestId('property-actions-quote')); + expect(res.queryByTestId('property-actions-description-quote')).toBeInTheDocument(); + userEvent.click(res.getByTestId('property-actions-description-quote')); await waitFor(() => { expect(builderArgs.handleManageQuote).toHaveBeenCalledWith('Security banana Issue'); diff --git a/x-pack/plugins/cases/public/components/user_actions/index.test.tsx b/x-pack/plugins/cases/public/components/user_actions/index.test.tsx index 2cfc535940f45..8923a45c48c2a 100644 --- a/x-pack/plugins/cases/public/components/user_actions/index.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/index.test.tsx @@ -197,13 +197,13 @@ describe(`UserActions`, () => { wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-ellipses"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-ellipses"]` ) .first() .simulate('click'); wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-pencil"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-pencil"]` ) .first() .simulate('click'); @@ -241,14 +241,14 @@ describe(`UserActions`, () => { wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-ellipses"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-ellipses"]` ) .first() .simulate('click'); wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-pencil"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-pencil"]` ) .first() .simulate('click'); @@ -293,12 +293,16 @@ describe(`UserActions`, () => { ); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-ellipses"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-ellipses"]` + ) .first() .simulate('click'); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-pencil"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-pencil"]` + ) .first() .simulate('click'); @@ -341,12 +345,16 @@ describe(`UserActions`, () => { expect(wrapper.find(`.euiMarkdownEditorTextArea`).text()).not.toContain(quoteableText); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-ellipses"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-ellipses"]` + ) .first() .simulate('click'); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-quote"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-quote"]` + ) .first() .simulate('click'); @@ -402,14 +410,14 @@ describe(`UserActions`, () => { wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-ellipses"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-ellipses"]` ) .first() .simulate('click'); wrapper .find( - `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-pencil"]` + `[data-test-subj="comment-create-action-${props.data.comments[0].id}"] [data-test-subj="property-actions-user-action-pencil"]` ) .first() .simulate('click'); @@ -463,12 +471,16 @@ describe(`UserActions`, () => { .simulate('change', { target: { value: newComment } }); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-ellipses"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-ellipses"]` + ) .first() .simulate('click'); wrapper - .find(`[data-test-subj="description-action"] [data-test-subj="property-actions-pencil"]`) + .find( + `[data-test-subj="description-action"] [data-test-subj="property-actions-description-pencil"]` + ) .first() .simulate('click'); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/alert_property_actions.test.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/alert_property_actions.test.tsx index dc8a57b8477f6..79636d52572ba 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/alert_property_actions.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/alert_property_actions.test.tsx @@ -34,26 +34,26 @@ describe('AlertPropertyActions', () => { it('renders the correct number of actions', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.getByTestId('property-actions-group').children.length).toBe(1); - expect(result.queryByTestId('property-actions-minusInCircle')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action-group').children.length).toBe(1); + expect(result.queryByTestId('property-actions-user-action-minusInCircle')).toBeInTheDocument(); }); it('renders the modal info correctly for one alert', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-minusInCircle')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-minusInCircle')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-minusInCircle')); + userEvent.click(result.getByTestId('property-actions-user-action-minusInCircle')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -66,14 +66,14 @@ describe('AlertPropertyActions', () => { it('renders the modal info correctly for multiple alert', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-minusInCircle')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-minusInCircle')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-minusInCircle')); + userEvent.click(result.getByTestId('property-actions-user-action-minusInCircle')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -86,14 +86,14 @@ describe('AlertPropertyActions', () => { it('remove alerts correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-minusInCircle')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-minusInCircle')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-minusInCircle')); + userEvent.click(result.getByTestId('property-actions-user-action-minusInCircle')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -107,13 +107,13 @@ describe('AlertPropertyActions', () => { appMock = createAppMockRenderer({ permissions: noCasesPermissions() }); const result = appMock.render(); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action')).not.toBeInTheDocument(); }); it('does show the property actions with only delete permissions', async () => { appMock = createAppMockRenderer({ permissions: onlyDeleteCasesPermission() }); const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.test.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.test.tsx index bfaa349caf46f..4a164c5e1fbda 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.test.tsx @@ -29,27 +29,27 @@ describe('DescriptionPropertyActions', () => { it('renders the correct number of actions', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-description')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-description-ellipses')); await waitForEuiPopoverOpen(); - expect(result.getByTestId('property-actions-group').children.length).toBe(2); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); - expect(result.queryByTestId('property-actions-quote')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-description-group').children.length).toBe(2); + expect(result.queryByTestId('property-actions-description-pencil')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-description-quote')).toBeInTheDocument(); }); it('edits the description correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-description')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-description-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-description-pencil')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-pencil')); + userEvent.click(result.getByTestId('property-actions-description-pencil')); expect(props.onEdit).toHaveBeenCalled(); }); @@ -57,14 +57,14 @@ describe('DescriptionPropertyActions', () => { it('quotes the description correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-description')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-description-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-quote')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-description-quote')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-quote')); + userEvent.click(result.getByTestId('property-actions-description-quote')); expect(props.onQuote).toHaveBeenCalled(); }); @@ -73,6 +73,6 @@ describe('DescriptionPropertyActions', () => { appMock = createAppMockRenderer({ permissions: noCasesPermissions() }); const result = appMock.render(); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-description')).not.toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.tsx index 5ef72a5590140..1b948f57e6448 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/description_property_actions.tsx @@ -45,7 +45,13 @@ const DescriptionPropertyActionsComponent: React.FC = ({ isLoading, onEdi ]; }, [permissions.update, permissions.create, onEdit, onQuote]); - return ; + return ( + + ); }; DescriptionPropertyActionsComponent.displayName = 'DescriptionPropertyActions'; diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.test.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.test.tsx index c7cfdb25bb359..ca310ab5121aa 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.test.tsx @@ -36,19 +36,19 @@ describe('UserActionPropertyActions', () => { const result = appMock.render(); expect(result.getByTestId('user-action-title-loading')).toBeInTheDocument(); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action')).not.toBeInTheDocument(); }); it('renders the property actions', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.getByTestId('property-actions-group').children.length).toBe(1); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action-group').children.length).toBe(1); + expect(result.queryByTestId('property-actions-user-action-pencil')).toBeInTheDocument(); }); it('does not render if properties are empty', async () => { @@ -56,7 +56,7 @@ describe('UserActionPropertyActions', () => { ); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action')).not.toBeInTheDocument(); expect(result.queryByTestId('user-action-title-loading')).not.toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.tsx index abf897404711a..975a8670ab096 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/property_actions.tsx @@ -13,9 +13,14 @@ import { PropertyActions } from '../../property_actions'; interface Props { isLoading: boolean; propertyActions: PropertyActionButtonProps[]; + customDataTestSubj?: string; } -const UserActionPropertyActionsComponent: React.FC = ({ isLoading, propertyActions }) => { +const UserActionPropertyActionsComponent: React.FC = ({ + isLoading, + propertyActions, + customDataTestSubj = 'user-action', +}) => { if (propertyActions.length === 0) { return null; } @@ -25,7 +30,10 @@ const UserActionPropertyActionsComponent: React.FC = ({ isLoading, proper {isLoading ? ( ) : ( - + )} ); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/registered_attachments_property_actions.test.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/registered_attachments_property_actions.test.tsx index a756f43893e03..9e391fa6e7042 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/registered_attachments_property_actions.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/registered_attachments_property_actions.test.tsx @@ -33,26 +33,26 @@ describe('RegisteredAttachmentsPropertyActions', () => { it('renders the correct number of actions', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.getByTestId('property-actions-group').children.length).toBe(1); - expect(result.queryByTestId('property-actions-trash')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action-group').children.length).toBe(1); + expect(result.queryByTestId('property-actions-user-action-trash')).toBeInTheDocument(); }); it('renders the modal info correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-trash')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-trash')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-trash')); + userEvent.click(result.getByTestId('property-actions-user-action-trash')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -65,14 +65,14 @@ describe('RegisteredAttachmentsPropertyActions', () => { it('remove attachments correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-trash')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-trash')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-trash')); + userEvent.click(result.getByTestId('property-actions-user-action-trash')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -86,13 +86,13 @@ describe('RegisteredAttachmentsPropertyActions', () => { appMock = createAppMockRenderer({ permissions: noCasesPermissions() }); const result = appMock.render(); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action')).not.toBeInTheDocument(); }); it('does show the property actions with only delete permissions', async () => { appMock = createAppMockRenderer({ permissions: onlyDeleteCasesPermission() }); const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cases/public/components/user_actions/property_actions/user_comment_property_actions.test.tsx b/x-pack/plugins/cases/public/components/user_actions/property_actions/user_comment_property_actions.test.tsx index 557dae707c20f..643662dfbc3f7 100644 --- a/x-pack/plugins/cases/public/components/user_actions/property_actions/user_comment_property_actions.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/property_actions/user_comment_property_actions.test.tsx @@ -35,28 +35,28 @@ describe('UserCommentPropertyActions', () => { it('renders the correct number of actions', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.getByTestId('property-actions-group').children.length).toBe(3); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); - expect(result.queryByTestId('property-actions-trash')).toBeInTheDocument(); - expect(result.queryByTestId('property-actions-quote')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action-group').children.length).toBe(3); + expect(result.queryByTestId('property-actions-user-action-pencil')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-trash')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-quote')).toBeInTheDocument(); }); it('edits the comment correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-pencil')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-pencil')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-pencil')); + userEvent.click(result.getByTestId('property-actions-user-action-pencil')); expect(props.onEdit).toHaveBeenCalled(); }); @@ -64,14 +64,14 @@ describe('UserCommentPropertyActions', () => { it('quotes the comment correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-quote')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-quote')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-quote')); + userEvent.click(result.getByTestId('property-actions-user-action-quote')); expect(props.onQuote).toHaveBeenCalled(); }); @@ -79,14 +79,14 @@ describe('UserCommentPropertyActions', () => { it('deletes the comment correctly', async () => { const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-ellipses')); + userEvent.click(result.getByTestId('property-actions-user-action-ellipses')); await waitForEuiPopoverOpen(); - expect(result.queryByTestId('property-actions-trash')).toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action-trash')).toBeInTheDocument(); - userEvent.click(result.getByTestId('property-actions-trash')); + userEvent.click(result.getByTestId('property-actions-user-action-trash')); await waitFor(() => { expect(result.queryByTestId('property-actions-confirm-modal')).toBeInTheDocument(); @@ -100,13 +100,13 @@ describe('UserCommentPropertyActions', () => { appMock = createAppMockRenderer({ permissions: noCasesPermissions() }); const result = appMock.render(); - expect(result.queryByTestId('property-actions')).not.toBeInTheDocument(); + expect(result.queryByTestId('property-actions-user-action')).not.toBeInTheDocument(); }); it('does show the property actions with only delete permissions', async () => { appMock = createAppMockRenderer({ permissions: onlyDeleteCasesPermission() }); const result = appMock.render(); - expect(result.getByTestId('property-actions')).toBeInTheDocument(); + expect(result.getByTestId('property-actions-user-action')).toBeInTheDocument(); }); }); diff --git a/x-pack/test/functional/services/cases/list.ts b/x-pack/test/functional/services/cases/list.ts index 1e420cd37368c..ced5c5acedca2 100644 --- a/x-pack/test/functional/services/cases/list.ts +++ b/x-pack/test/functional/services/cases/list.ts @@ -81,9 +81,6 @@ export function CasesTableServiceProvider( rows = await find.allByCssSelector('[data-test-subj*="cases-table-row-"', 100); if (rows.length > 0) { await this.bulkDeleteAllCases(); - // wait for a second - await new Promise((r) => setTimeout(r, 1000)); - await header.waitUntilLoadingHasFinished(); } } while (rows.length > 0); }, diff --git a/x-pack/test/functional/services/cases/single_case_view.ts b/x-pack/test/functional/services/cases/single_case_view.ts index bd9377ebd5abd..b8bea0841a605 100644 --- a/x-pack/test/functional/services/cases/single_case_view.ts +++ b/x-pack/test/functional/services/cases/single_case_view.ts @@ -20,14 +20,12 @@ export function CasesSingleViewServiceProvider({ getService, getPageObject }: Ft return { async deleteCase() { - const caseActions = await testSubjects.findDescendant( - 'property-actions-ellipses', - await testSubjects.find('case-view-actions') - ); + await retry.try(async () => { + await testSubjects.click('property-actions-case-ellipses'); + await testSubjects.existOrFail('property-actions-case-trash', { timeout: 100 }); + }); - await caseActions.click(); - await testSubjects.existOrFail('property-actions-trash'); - await common.clickAndValidate('property-actions-trash', 'confirmModalConfirmButton'); + await common.clickAndValidate('property-actions-case-trash', 'confirmModalConfirmButton'); await testSubjects.click('confirmModalConfirmButton'); await header.waitUntilLoadingHasFinished(); }, diff --git a/x-pack/test/functional_with_es_ssl/apps/cases/deletion.ts b/x-pack/test/functional_with_es_ssl/apps/cases/deletion.ts index 0f8d551ce86aa..4179c549484fb 100644 --- a/x-pack/test/functional_with_es_ssl/apps/cases/deletion.ts +++ b/x-pack/test/functional_with_es_ssl/apps/cases/deletion.ts @@ -17,8 +17,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const testSubjects = getService('testSubjects'); const cases = getService('cases'); - // Failing: See https://github.com/elastic/kibana/issues/145271 - describe.skip('cases deletion sub privilege', () => { + describe('cases deletion sub privilege', () => { before(async () => { await createUsersAndRoles(getService, users, roles); await PageObjects.security.forceLogout(); @@ -101,7 +100,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it(`User ${user.username} cannot delete a case while on a specific case page`, async () => { - await testSubjects.missingOrFail('case-view-actions'); + await testSubjects.click('property-actions-case-ellipses'); + await testSubjects.missingOrFail('property-actions-case-trash'); }); }); diff --git a/x-pack/test/functional_with_es_ssl/apps/cases/view_case.ts b/x-pack/test/functional_with_es_ssl/apps/cases/view_case.ts index 876857721966f..e808aaa8a4463 100644 --- a/x-pack/test/functional_with_es_ssl/apps/cases/view_case.ts +++ b/x-pack/test/functional_with_es_ssl/apps/cases/view_case.ts @@ -14,7 +14,7 @@ import { createUsersAndRoles, deleteUsersAndRoles, } from '../../../cases_api_integration/common/lib/authentication'; -import { users, roles, casesAllUser } from './common'; +import { users, roles, casesAllUser, casesAllUser2 } from './common'; export default ({ getPageObject, getService }: FtrProviderContext) => { const header = getPageObject('header'); @@ -282,22 +282,14 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { await header.waitUntilLoadingHasFinished(); - const propertyActions = await find.allByCssSelector( - '[data-test-subj*="property-actions-ellipses"]' - ); - - propertyActions[propertyActions.length - 1].click(); + await testSubjects.click('property-actions-user-action-ellipses'); await header.waitUntilLoadingHasFinished(); - const editAction = await find.byCssSelector( - '[data-test-subj*="property-actions-pencil"]' - ); + await testSubjects.click('property-actions-user-action-pencil'); await header.waitUntilLoadingHasFinished(); - await editAction.click(); - const editCommentTextArea = await find.byCssSelector( '[data-test-subj*="user-action-markdown-form"] textarea.euiMarkdownEditorTextArea' ); @@ -315,22 +307,14 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { }); it('shows unsaved description message when page is refreshed', async () => { - const propertyActions = await find.allByCssSelector( - '[data-test-subj*="property-actions-ellipses"]' - ); - - propertyActions[1].click(); + await testSubjects.click('property-actions-description-ellipses'); await header.waitUntilLoadingHasFinished(); - const editAction = await find.byCssSelector( - '[data-test-subj*="property-actions-pencil"]' - ); + await testSubjects.click('property-actions-description-pencil'); await header.waitUntilLoadingHasFinished(); - await editAction.click(); - const editCommentTextArea = await find.byCssSelector( '[data-test-subj*="user-action-markdown-form"] textarea.euiMarkdownEditorTextArea' ); @@ -379,7 +363,7 @@ export default ({ getPageObject, getService }: FtrProviderContext) => { describe('Assignees field', () => { before(async () => { await createUsersAndRoles(getService, users, roles); - await cases.api.activateUserProfiles([casesAllUser]); + await cases.api.activateUserProfiles([casesAllUser, casesAllUser2]); }); after(async () => { From 6de805636feef96fad6686ba37d58c9a3dd319e2 Mon Sep 17 00:00:00 2001 From: Sloane Perrault Date: Mon, 6 Feb 2023 15:44:41 -0500 Subject: [PATCH 015/134] [Enterprise Search] Engines Overview with indices and documents count stats (#149980) ## Summary Adds Engine overview page. Includes panel with document and index counts in stats components image ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### For maintainers - [x] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../enterprise_search/common/types/engines.ts | 9 +- ...ngine_field_capabilities_api_logic.test.ts | 31 +++++ ...tch_engine_field_capabilities_api_logic.ts | 34 ++++++ .../components/engine/engine_overview.tsx | 109 ++++++++++++++++++ .../engine/engine_overview_logic.test.ts | 39 +++++++ .../engine/engine_overview_logic.ts | 89 ++++++++++++++ .../components/engine/engine_view.tsx | 6 + .../components/engine/engine_view_logic.ts | 8 +- .../routes/enterprise_search/engines.ts | 10 ++ 9 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.ts diff --git a/x-pack/plugins/enterprise_search/common/types/engines.ts b/x-pack/plugins/enterprise_search/common/types/engines.ts index 3926be635c5bd..8a0ef6d216a20 100644 --- a/x-pack/plugins/enterprise_search/common/types/engines.ts +++ b/x-pack/plugins/enterprise_search/common/types/engines.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { HealthStatus } from '@elastic/elasticsearch/lib/api/types'; +import { HealthStatus, FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; export interface EnterpriseSearchEnginesResponse { meta: { @@ -37,3 +37,10 @@ export interface EnterpriseSearchEngineIndex { name: string; source: 'api' | 'connector' | 'crawler'; } + +export interface EnterpriseSearchEngineFieldCapabilities { + created: string; + field_capabilities: FieldCapsResponse; + name: string; + updated: string; +} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.test.ts new file mode 100644 index 0000000000000..5eb05fbed4c8f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockHttpValues } from '../../../__mocks__/kea_logic'; + +import { nextTick } from '@kbn/test-jest-helpers'; + +import { fetchEngineFieldCapabilities } from './fetch_engine_field_capabilities_api_logic'; + +describe('FetchEngineFieldCapabilitiesApiLogic', () => { + const { http } = mockHttpValues; + beforeEach(() => { + jest.clearAllMocks(); + }); + describe('fetchEngineFieldCapabilities', () => { + it('requests the field_capabilities api', async () => { + const promise = Promise.resolve({ result: 'result' }); + http.get.mockReturnValue(promise); + const result = fetchEngineFieldCapabilities({ engineName: 'foobar' }); + await nextTick(); + expect(http.get).toHaveBeenCalledWith( + '/internal/enterprise_search/engines/foobar/field_capabilities' + ); + await expect(result).resolves.toEqual({ result: 'result' }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.ts new file mode 100644 index 0000000000000..7f5d196a5a02d --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/engines/fetch_engine_field_capabilities_api_logic.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EnterpriseSearchEngineFieldCapabilities } from '../../../../../common/types/engines'; +import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { HttpLogic } from '../../../shared/http'; + +export interface FetchEngineFieldCapabilitiesApiParams { + engineName: string; +} + +export type FetchEngineFieldCapabilitiesApiResponse = EnterpriseSearchEngineFieldCapabilities; + +export const fetchEngineFieldCapabilities = async ({ + engineName, +}: FetchEngineFieldCapabilitiesApiParams): Promise => { + const route = `/internal/enterprise_search/engines/${engineName}/field_capabilities`; + + return await HttpLogic.values.http.get(route); +}; + +export const FetchEngineFieldCapabilitiesApiLogic = createApiLogic( + ['fetch_engine_field_capabilities_api_logic'], + fetchEngineFieldCapabilities +); + +export type FetchEngineFieldCapabilitiesApiLogicActions = Actions< + FetchEngineFieldCapabilitiesApiParams, + FetchEngineFieldCapabilitiesApiResponse +>; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview.tsx new file mode 100644 index 0000000000000..83312c8290e21 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview.tsx @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useValues } from 'kea'; + +import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiPanel, EuiStat } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { generateEncodedPath } from '../../../shared/encode_path_params'; +import { EuiLinkTo } from '../../../shared/react_router_helpers'; +import { EngineViewTabs, ENGINE_TAB_PATH } from '../../routes'; +import { EnterpriseSearchEnginesPageTemplate } from '../layout/engines_page_template'; + +import { EngineOverviewLogic } from './engine_overview_logic'; +import { EngineViewHeaderActions } from './engine_view_header_actions'; + +export const EngineOverview: React.FC = () => { + const { engineName, indicesCount, documentsCount, fieldsCount, isLoadingEngine } = + useValues(EngineOverviewLogic); + + return ( + ], + }} + engineName={engineName} + > + <> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts new file mode 100644 index 0000000000000..0a012a07fd9da --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter } from '../../../__mocks__/kea_logic'; + +import { Status } from '../../../../../common/types/api'; + +import { EngineOverviewLogic, EngineOverviewValues } from './engine_overview_logic'; + +const DEFAULT_VALUES: EngineOverviewValues = { + documentsCount: 0, + engineData: undefined, + engineFieldCapabilitiesApiStatus: Status.IDLE, + engineFieldCapabilitiesData: undefined, + engineName: '', + fieldsCount: 0, + indices: [], + indicesCount: 0, + isLoadingEngine: true, +}; + +describe('EngineOverviewLogic', () => { + const { mount } = new LogicMounter(EngineOverviewLogic); + + beforeEach(() => { + jest.clearAllMocks(); + jest.useRealTimers(); + + mount(); + }); + + it('has expected default values', () => { + expect(EngineOverviewLogic.values).toEqual(DEFAULT_VALUES); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.ts new file mode 100644 index 0000000000000..f6a8508d131b1 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { Status } from '../../../../../common/types/api'; +import { EnterpriseSearchEngineIndex } from '../../../../../common/types/engines'; + +import { FetchEngineFieldCapabilitiesApiLogic } from '../../api/engines/fetch_engine_field_capabilities_api_logic'; + +import { EngineNameLogic } from './engine_name_logic'; +import { EngineViewLogic } from './engine_view_logic'; + +export interface EngineOverviewActions { + fetchEngineFieldCapabilities: typeof FetchEngineFieldCapabilitiesApiLogic.actions.makeRequest; +} +export interface EngineOverviewValues { + documentsCount: number; + engineData: typeof EngineViewLogic.values.engineData; + engineFieldCapabilitiesApiStatus: typeof FetchEngineFieldCapabilitiesApiLogic.values.status; + engineFieldCapabilitiesData: typeof FetchEngineFieldCapabilitiesApiLogic.values.data; + engineName: typeof EngineNameLogic.values.engineName; + fieldsCount: number; + indices: EnterpriseSearchEngineIndex[]; + indicesCount: number; + isLoadingEngine: typeof EngineViewLogic.values.isLoadingEngine; +} + +export const EngineOverviewLogic = kea>({ + actions: {}, + connect: { + actions: [ + EngineNameLogic, + ['setEngineName'], + FetchEngineFieldCapabilitiesApiLogic, + ['makeRequest as fetchEngineFieldCapabilities'], + ], + values: [ + EngineNameLogic, + ['engineName'], + EngineViewLogic, + ['engineData', 'isLoadingEngine'], + FetchEngineFieldCapabilitiesApiLogic, + ['data as engineFieldCapabilitiesData', 'status as engineFieldCapabilitiesApiStatus'], + ], + }, + events: ({ actions, values }) => ({ + afterMount: () => { + if (values.engineFieldCapabilitiesApiStatus !== Status.SUCCESS && !!values.engineName) { + actions.fetchEngineFieldCapabilities({ + engineName: values.engineName, + }); + } + }, + }), + listeners: ({ actions }) => ({ + setEngineName: ({ engineName }) => { + actions.fetchEngineFieldCapabilities({ engineName }); + }, + }), + path: ['enterprise_search', 'content', 'engine_overview_logic'], + reducers: {}, + selectors: ({ selectors }) => ({ + documentsCount: [ + () => [selectors.indices], + (indices: EngineOverviewValues['indices']) => + indices.reduce((sum, { count }) => sum + count, 0), + ], + fieldsCount: [ + () => [selectors.engineFieldCapabilitiesData], + (engineFieldCapabilitiesData: EngineOverviewValues['engineFieldCapabilitiesData']) => + Object.values(engineFieldCapabilitiesData?.field_capabilities?.fields ?? {}).filter( + (value) => !Object.values(value).some((field) => !!field.metadata_field) + ).length, + ], + indices: [ + () => [selectors.engineData], + (engineData: EngineOverviewValues['engineData']) => engineData?.indices ?? [], + ], + indicesCount: [ + () => [selectors.indices], + (indices: EngineOverviewValues['indices']) => indices.length, + ], + }), +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view.tsx index e6057a2d1b2bd..0c65e8008f74e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view.tsx @@ -21,6 +21,7 @@ import { EnterpriseSearchEnginesPageTemplate } from '../layout/engines_page_temp import { EngineAPI } from './engine_api/engine_api'; import { EngineError } from './engine_error'; import { EngineIndices } from './engine_indices'; +import { EngineOverview } from './engine_overview'; import { EngineViewHeaderActions } from './engine_view_header_actions'; import { EngineViewLogic } from './engine_view_logic'; import { EngineHeaderDocsAction } from './header_docs_action'; @@ -66,6 +67,11 @@ export const EngineView: React.FC = () => { ) : null} + >({ + actions: { + closeDeleteEngineModal: true, + openDeleteEngineModal: true, + }, connect: { actions: [ FetchEngineApiLogic, @@ -53,10 +57,6 @@ export const EngineViewLogic = kea ({ deleteSuccess: () => { actions.closeDeleteEngineModal(); diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts index c64c86b35fadb..2a0469d77cd48 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts @@ -128,4 +128,14 @@ export function registerEnginesRoutes({ path: '/api/engines/:engine_name/_search', }) ); + + router.get( + { + path: '/internal/enterprise_search/engines/{engine_name}/field_capabilities', + validate: { params: schema.object({ engine_name: schema.string() }) }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/api/engines/:engine_name/field_capabilities', + }) + ); } From 6e5fe42aaa607e71a3c15824cdb8306e1255f955 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Mon, 6 Feb 2023 13:59:20 -0700 Subject: [PATCH 016/134] [Security solution] Network page maps, fix bad side effects (#149368) --- .../explore/containers/fields/index.test.ts | 51 +++++++++---------- .../public/explore/containers/fields/index.ts | 30 +++++++---- .../embeddables/embedded_map.test.tsx | 19 ++++--- .../components/embeddables/embedded_map.tsx | 31 +++++------ 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/security_solution/public/explore/containers/fields/index.test.ts b/x-pack/plugins/security_solution/public/explore/containers/fields/index.test.ts index b10801ed41364..03084ef42dd8d 100644 --- a/x-pack/plugins/security_solution/public/explore/containers/fields/index.test.ts +++ b/x-pack/plugins/security_solution/public/explore/containers/fields/index.test.ts @@ -6,30 +6,15 @@ */ import { useKibana } from '../../../common/lib/kibana'; import { useIsFieldInIndexPattern } from '.'; +import { renderHook } from '@testing-library/react-hooks'; +import { getRequiredMapsFields } from '../../network/components/embeddables/map_config'; jest.mock('../../../common/lib/kibana'); +jest.mock('../../network/components/embeddables/map_config'); const mockUseKibana = useKibana as jest.Mock; describe('useIsFieldInIndexPattern', () => { - beforeAll(() => { - mockUseKibana.mockReturnValue({ - services: { - data: { - dataViews: { - getFieldsForWildcard: () => [], - }, - }, - }, - }); - }); beforeEach(() => { jest.clearAllMocks(); - }); - it('returns false when no fields in field list exist in the index pattern', async () => { - const isFieldInIndexPattern = useIsFieldInIndexPattern(); - const res = await isFieldInIndexPattern('index-pattern-*', ['fields.list']); - expect(res).toEqual(false); - }); - it('returns false when some but not all fields in field list exist in the index pattern', async () => { mockUseKibana.mockReturnValue({ services: { http: {}, @@ -40,23 +25,37 @@ describe('useIsFieldInIndexPattern', () => { }, }, }); - const isFieldInIndexPattern = useIsFieldInIndexPattern(); - const res = await isFieldInIndexPattern('index-pattern-*', ['fields.list', 'another']); - expect(res).toEqual(false); + (getRequiredMapsFields as jest.Mock).mockReturnValue(['fields.list']); }); - it('returns true when all fields in field list exist in the index pattern', async () => { + it('returns false when no fields in field list exist in the index pattern', async () => { mockUseKibana.mockReturnValue({ services: { - http: {}, data: { dataViews: { - getFieldsForWildcard: () => [{ name: 'fields.list' }], + getFieldsForWildcard: () => [], }, }, }, }); - const isFieldInIndexPattern = useIsFieldInIndexPattern(); - const res = await isFieldInIndexPattern('index-pattern-*', ['fields.list']); + const { + result: { current: isFieldInIndexPattern }, + } = renderHook(useIsFieldInIndexPattern); + const res = await isFieldInIndexPattern('index-pattern-*'); + expect(res).toEqual(false); + }); + it('returns false when some but not all fields in field list exist in the index pattern', async () => { + (getRequiredMapsFields as jest.Mock).mockReturnValue(['fields.list', 'another']); + const { + result: { current: isFieldInIndexPattern }, + } = renderHook(useIsFieldInIndexPattern); + const res = await isFieldInIndexPattern('index-pattern-*'); + expect(res).toEqual(false); + }); + it('returns true when all fields in field list exist in the index pattern', async () => { + const { + result: { current: isFieldInIndexPattern }, + } = renderHook(useIsFieldInIndexPattern); + const res = await isFieldInIndexPattern('index-pattern-*'); expect(res).toEqual(true); }); }); diff --git a/x-pack/plugins/security_solution/public/explore/containers/fields/index.ts b/x-pack/plugins/security_solution/public/explore/containers/fields/index.ts index 81c6765e409c6..b120956ab24c4 100644 --- a/x-pack/plugins/security_solution/public/explore/containers/fields/index.ts +++ b/x-pack/plugins/security_solution/public/explore/containers/fields/index.ts @@ -5,18 +5,30 @@ * 2.0. */ +import { useMemo } from 'react'; +import memoizeOne from 'memoize-one'; +import { getRequiredMapsFields } from '../../network/components/embeddables/map_config'; import { useKibana } from '../../../common/lib/kibana'; -type FieldValidationCheck = (pattern: string, fieldsList: string[]) => Promise; +type FieldValidationCheck = (pattern: string) => Promise; export const useIsFieldInIndexPattern = (): FieldValidationCheck => { const { dataViews } = useKibana().services.data; - return async (pattern: string, fieldsList: string[]) => { - const fields = await dataViews.getFieldsForWildcard({ - pattern, - fields: fieldsList, - }); - const fieldNames = fields.map((f) => f.name); - return fieldsList.every((field) => fieldNames.includes(field)); - }; + + return useMemo( + () => + memoizeOne( + async (pattern: string) => { + const fieldsList = getRequiredMapsFields(pattern); + const fields = await dataViews.getFieldsForWildcard({ + pattern, + fields: fieldsList, + }); + const fieldNames = fields.map((f) => f.name); + return fieldsList.every((field) => fieldNames.includes(field)); + }, + (newArgs, lastArgs) => newArgs[0] === lastArgs[0] + ), + [dataViews] + ); }; diff --git a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.test.tsx b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.test.tsx index 4c938d5e92d78..4a78cd8939921 100644 --- a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.test.tsx +++ b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.test.tsx @@ -58,9 +58,9 @@ const mockGetStorage = jest.fn(); const mockSetStorage = jest.fn(); const setQuery: jest.Mock = jest.fn(); const filebeatDataView = { id: '6f1eeb50-023d-11eb-bcb6-6ba0578012a9', title: 'filebeat-*' }; -const auditbeatDataView = { id: '28995490-023d-11eb-bcb6-6ba0578012a9', title: 'auditbeat-*' }; +const packetbeatDataView = { id: '28995490-023d-11eb-bcb6-6ba0578012a9', title: 'packetbeat-*' }; const mockSelector = { - kibanaDataViews: [filebeatDataView, auditbeatDataView], + kibanaDataViews: [filebeatDataView, packetbeatDataView], }; const embeddableValue = { destroyed: false, @@ -102,9 +102,9 @@ describe('EmbeddedMapComponent', () => { setQuery.mockClear(); mockGetStorage.mockReturnValue(true); jest.spyOn(redux, 'useSelector').mockReturnValue(mockSelector); - mockUseSourcererDataView.mockReturnValue({ selectedPatterns: ['filebeat-*', 'packetbeat-*'] }); + mockUseSourcererDataView.mockReturnValue({ selectedPatterns: ['filebeat-*', 'auditbeat-*'] }); mockCreateEmbeddable.mockResolvedValue(embeddableValue); - mockUseIsFieldInIndexPattern.mockReturnValue(() => [true, true]); + mockUseIsFieldInIndexPattern.mockReturnValue(() => true); }); afterEach(() => { @@ -215,9 +215,9 @@ describe('EmbeddedMapComponent', () => { }); test('On rerender with new selected patterns, selects existing Kibana data views that match any selected index pattern', async () => { - mockUseSourcererDataView - .mockReturnValueOnce({ selectedPatterns: ['filebeat-*', 'packetbeat-*'] }) - .mockReturnValue({ selectedPatterns: ['filebeat-*', 'auditbeat-*'] }); + mockUseSourcererDataView.mockReturnValue({ + selectedPatterns: ['filebeat-*', 'auditbeat-*'], + }); const { rerender } = render( @@ -227,6 +227,9 @@ describe('EmbeddedMapComponent', () => { const dataViewArg = (getLayerList as jest.Mock).mock.calls[0][0]; expect(dataViewArg).toEqual([filebeatDataView]); }); + mockUseSourcererDataView.mockReturnValue({ + selectedPatterns: ['filebeat-*', 'packetbeat-*'], + }); rerender( @@ -235,7 +238,7 @@ describe('EmbeddedMapComponent', () => { await waitFor(() => { // data view is updated with the returned embeddable.setLayerList callback, which is passesd getLayerList(dataViews) const dataViewArg = (getLayerList as jest.Mock).mock.calls[1][0]; - expect(dataViewArg).toEqual([filebeatDataView, auditbeatDataView]); + expect(dataViewArg).toEqual([filebeatDataView, packetbeatDataView]); }); }); }); diff --git a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx index 14215000c5611..8e94720398d70 100644 --- a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx +++ b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx @@ -5,15 +5,17 @@ * 2.0. */ +// embedded map v2 + import { EuiAccordion, EuiLink, EuiText } from '@elastic/eui'; import React, { useCallback, useEffect, useState, useMemo } from 'react'; import { createHtmlPortalNode, InPortal } from 'react-reverse-portal'; import styled, { css } from 'styled-components'; - import type { Filter, Query } from '@kbn/es-query'; import type { ErrorEmbeddable } from '@kbn/embeddable-plugin/public'; import { isErrorEmbeddable } from '@kbn/embeddable-plugin/public'; import type { MapEmbeddable } from '@kbn/maps-plugin/public/embeddable'; +import { isEqual } from 'lodash/fp'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import { useIsFieldInIndexPattern } from '../../../containers/fields'; import { Loader } from '../../../../common/components/loader'; @@ -24,7 +26,7 @@ import { IndexPatternsMissingPrompt } from './index_patterns_missing_prompt'; import { MapToolTip } from './map_tool_tip/map_tool_tip'; import * as i18n from './translations'; import { useKibana } from '../../../../common/lib/kibana'; -import { getLayerList, getRequiredMapsFields } from './map_config'; +import { getLayerList } from './map_config'; import { sourcererSelectors } from '../../../../common/store/sourcerer'; import type { SourcererDataView } from '../../../../common/store/sourcerer/model'; import { SourcererScopeName } from '../../../../common/store/sourcerer/model'; @@ -123,16 +125,7 @@ export const EmbeddedMapComponent = ({ const isFieldInIndexPattern = useIsFieldInIndexPattern(); const [mapDataViews, setMapDataViews] = useState([]); - - const availableDataViews = useMemo(() => { - const dataViews = kibanaDataViews.filter((dataView) => - selectedPatterns.includes(dataView.title) - ); - if (selectedPatterns.length > 0 && dataViews.length === 0) { - setIsIndexError(true); - } - return dataViews; - }, [kibanaDataViews, selectedPatterns]); + const [availableDataViews, setAvailableDataViews] = useState([]); useEffect(() => { let canceled = false; @@ -140,9 +133,7 @@ export const EmbeddedMapComponent = ({ const fetchData = async () => { try { const apiResponse = await Promise.all( - availableDataViews.map(async ({ title }) => - isFieldInIndexPattern(title, getRequiredMapsFields(title)) - ) + availableDataViews.map(async ({ title }) => isFieldInIndexPattern(title)) ); // ensures only index patterns with maps fields are passed const goodDataViews = availableDataViews.filter((_, i) => apiResponse[i] ?? false); @@ -165,6 +156,16 @@ export const EmbeddedMapComponent = ({ }; }, [addError, availableDataViews, isFieldInIndexPattern]); + useEffect(() => { + const dataViews = kibanaDataViews.filter((dataView) => + selectedPatterns.includes(dataView.title) + ); + if (selectedPatterns.length > 0 && dataViews.length === 0) { + setIsIndexError(true); + } + setAvailableDataViews((prevViews) => (isEqual(prevViews, dataViews) ? prevViews : dataViews)); + }, [kibanaDataViews, selectedPatterns]); + // This portalNode provided by react-reverse-portal allows us re-parent the MapToolTip within our // own component tree instead of the embeddables (default). This is necessary to have access to // the Redux store, theme provider, etc, which is required to register and un-register the draggable From 0188a895aa7f0021588d2da98f9d1b63bb0daa25 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:06:27 -0500 Subject: [PATCH 017/134] skip failing test suite (#134517) --- .../reporting_and_security/usage/api_counters.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts b/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts index 93b7d4d71fafb..6995436c47c50 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts @@ -17,7 +17,8 @@ export default function ({ getService }: FtrProviderContext) { const usageAPI = getService('usageAPI'); const reportingAPI = getService('reportingAPI'); - describe(`Usage Counters`, () => { + // Failing: See https://github.com/elastic/kibana/issues/134517 + describe.skip(`Usage Counters`, () => { before(async () => { await esArchiver.emptyKibanaIndex(); await reportingAPI.initEcommerce(); From c37e2542cf2cd2f2b4d1258b7d78f762e9176230 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:12:19 -0500 Subject: [PATCH 018/134] skip failing test suite (#149942) --- .../reporting_and_security/usage/api_counters.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts b/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts index 6995436c47c50..e490d88820756 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/usage/api_counters.ts @@ -18,6 +18,7 @@ export default function ({ getService }: FtrProviderContext) { const reportingAPI = getService('reportingAPI'); // Failing: See https://github.com/elastic/kibana/issues/134517 + // Failing: See https://github.com/elastic/kibana/issues/149942 describe.skip(`Usage Counters`, () => { before(async () => { await esArchiver.emptyKibanaIndex(); From b37eefce2d1988015f309de29e2d3971d23defa1 Mon Sep 17 00:00:00 2001 From: Ersin Erdal <92688503+ersin-erdal@users.noreply.github.com> Date: Mon, 6 Feb 2023 22:12:33 +0100 Subject: [PATCH 019/134] Make "for each alert" option default for all the rule types (#150344) As some of the users are using a template to fill out the message input with `for each alert` params, having `summary of alerts` option default causes some problems. Therefore, this PR intends to make "for each alert" option default for all the rule types. --- .../action_connector_form/action_form.tsx | 10 ++-- .../action_notify_when.test.tsx | 19 +++---- .../action_notify_when.tsx | 51 ++----------------- .../action_type_form.test.tsx | 14 ++--- .../sections/rule_form/rule_reducer.ts | 4 +- .../constants/action_frequency_types.ts | 8 +-- 6 files changed, 26 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_form.tsx index 900d2339ec87e..cc2810897bb08 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_form.tsx @@ -35,11 +35,7 @@ import { ActionTypeForm } from './action_type_form'; import { AddConnectorInline } from './connector_add_inline'; import { actionTypeCompare } from '../../lib/action_type_compare'; import { checkActionFormActionTypeEnabled } from '../../lib/check_action_type_enabled'; -import { - DEFAULT_FREQUENCY_WITH_SUMMARY, - DEFAULT_FREQUENCY_WITHOUT_SUMMARY, - VIEW_LICENSE_OPTIONS_LINK, -} from '../../../common/constants'; +import { DEFAULT_FREQUENCY, VIEW_LICENSE_OPTIONS_LINK } from '../../../common/constants'; import { useKibana } from '../../../common/lib/kibana'; import { ConnectorAddModal } from '.'; import { suspendedComponentWithProps } from '../../lib/suspended_component_with_props'; @@ -224,7 +220,7 @@ export const ActionForm = ({ actionTypeId: actionTypeModel.id, group: defaultActionGroupId, params: {}, - frequency: hasSummary ? DEFAULT_FREQUENCY_WITH_SUMMARY : DEFAULT_FREQUENCY_WITHOUT_SUMMARY, + frequency: DEFAULT_FREQUENCY, }); setActionIdByIndex(actionTypeConnectors[0].id, actions.length - 1); } @@ -236,7 +232,7 @@ export const ActionForm = ({ actionTypeId: actionTypeModel.id, group: defaultActionGroupId, params: {}, - frequency: hasSummary ? DEFAULT_FREQUENCY_WITH_SUMMARY : DEFAULT_FREQUENCY_WITHOUT_SUMMARY, + frequency: DEFAULT_FREQUENCY, }); setActionIdByIndex(actions.length.toString(), actions.length - 1); setEmptyActionsIds([...emptyActionsIds, actions.length.toString()]); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.test.tsx index 8fd286784824c..8186a7edbf09d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.test.tsx @@ -12,14 +12,11 @@ import { act } from 'react-dom/test-utils'; import { RuleAction } from '../../../types'; import { ActionNotifyWhen } from './action_notify_when'; import { RuleNotifyWhen } from '@kbn/alerting-plugin/common'; -import { - DEFAULT_FREQUENCY_WITHOUT_SUMMARY, - DEFAULT_FREQUENCY_WITH_SUMMARY, -} from '../../../common/constants'; +import { DEFAULT_FREQUENCY } from '../../../common/constants'; describe('action_notify_when', () => { async function setup( - frequency: RuleAction['frequency'] = DEFAULT_FREQUENCY_WITH_SUMMARY, + frequency: RuleAction['frequency'] = DEFAULT_FREQUENCY, hasSummary: boolean = true ) { const wrapper = mountWithIntl( @@ -50,15 +47,15 @@ describe('action_notify_when', () => { '[data-test-subj="summaryOrPerRuleSelect"]' ); expect(summaryOrPerRuleSelect.exists()).toBeTruthy(); - expect(summaryOrPerRuleSelect.first().props()['aria-label']).toEqual('Summary of alerts'); + expect(summaryOrPerRuleSelect.first().props()['aria-label']).toEqual('For each alert'); const notifyWhenSelect = wrapperDefault.find('[data-test-subj="notifyWhenSelect"]'); expect(notifyWhenSelect.exists()).toBeTruthy(); expect((notifyWhenSelect.first().props() as EuiSuperSelectProps<''>).valueOfSelected).toEqual( - RuleNotifyWhen.ACTIVE + RuleNotifyWhen.CHANGE ); } - const wrapperForEach = await setup(DEFAULT_FREQUENCY_WITHOUT_SUMMARY); + const wrapperForEach = await setup(DEFAULT_FREQUENCY); { const summaryOrPerRuleSelect = wrapperForEach.find( '[data-test-subj="summaryOrPerRuleSelect"]' @@ -73,7 +70,7 @@ describe('action_notify_when', () => { ); } const wrapperSummaryThrottle = await setup({ - ...DEFAULT_FREQUENCY_WITH_SUMMARY, + ...DEFAULT_FREQUENCY, throttle: '5h', notifyWhen: RuleNotifyWhen.THROTTLE, }); @@ -82,7 +79,7 @@ describe('action_notify_when', () => { '[data-test-subj="summaryOrPerRuleSelect"]' ); expect(summaryOrPerRuleSelect.exists()).toBeTruthy(); - expect(summaryOrPerRuleSelect.first().props()['aria-label']).toEqual('Summary of alerts'); + expect(summaryOrPerRuleSelect.first().props()['aria-label']).toEqual('For each alert'); const notifyWhenSelect = wrapperSummaryThrottle.find('[data-test-subj="notifyWhenSelect"]'); expect(notifyWhenSelect.exists()).toBeTruthy(); @@ -99,7 +96,7 @@ describe('action_notify_when', () => { }); it('hides the summary selector when hasSummary is false', async () => { - const wrapper = await setup(DEFAULT_FREQUENCY_WITHOUT_SUMMARY, false); + const wrapper = await setup(DEFAULT_FREQUENCY, false); const summaryOrPerRuleSelect = wrapper.find('[data-test-subj="summaryOrPerRuleSelect"]'); expect(summaryOrPerRuleSelect.exists()).toBeFalsy(); }); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.tsx index 5735568df592d..5295fcc08131d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_notify_when.tsx @@ -29,10 +29,7 @@ import { some, filter, map } from 'fp-ts/lib/Option'; import { pipe } from 'fp-ts/lib/pipeable'; import { getTimeOptions } from '../../../common/lib/get_time_options'; import { RuleNotifyWhenType, RuleAction } from '../../../types'; -import { - DEFAULT_FREQUENCY_WITH_SUMMARY, - DEFAULT_FREQUENCY_WITHOUT_SUMMARY, -} from '../../../common/constants'; +import { DEFAULT_FREQUENCY } from '../../../common/constants'; export const NOTIFY_WHEN_OPTIONS: Array> = [ { @@ -135,7 +132,7 @@ interface ActionNotifyWhenProps { export const ActionNotifyWhen = ({ hasSummary, - frequency = hasSummary ? DEFAULT_FREQUENCY_WITH_SUMMARY : DEFAULT_FREQUENCY_WITHOUT_SUMMARY, + frequency = DEFAULT_FREQUENCY, throttle, throttleUnit, onNotifyWhenChange, @@ -146,32 +143,7 @@ export const ActionNotifyWhen = ({ }: ActionNotifyWhenProps) => { const [showCustomThrottleOpts, setShowCustomThrottleOpts] = useState(false); const [notifyWhenValue, setNotifyWhenValue] = useState( - hasSummary - ? DEFAULT_FREQUENCY_WITH_SUMMARY.notifyWhen - : DEFAULT_FREQUENCY_WITHOUT_SUMMARY.notifyWhen - ); - - // Track whether the user has changed the notify when value from default. This is necessary because the - // "default" notifyWhen value for summary: true is the second menu item for summary: false. We want the UX to be: - // Case A - // - User opens the form with summary: false, notifyWhen: CHANGE - // - User switches to summary: true, necessitating a switch to notifyWhen: ACTIVE - // - User doesn't touch notifyWhen: ACTIVE, switches back to summary: false. notifyWhen should switch to CHANGE, the 1st menu option - // Case B - // - User opens the form with summary: false, notifyWhen: ACTIVE (not the "default") - // - User switches to summary: true - // - User switches back to summary: false. notifyWhen stays ACTIVE - // Case C - // - User opens the form with summary: true, notifyWhen: ACTIVE (the "default") - // - User doesn't change notifyWhen, just sets summary: false. notifyWhen should switch to CHANGE - // Case D - // - User opens the form with summary: true, notifyWhen: THROTTLE, or summary: false, notifyWhen: !CHANGE - // - When user changes summary, leave notifyWhen unchanged - const [notifyWhenValueChangedFromDefault, setNotifyWhenValueChangedFromDefault] = useState( - // Check if the initial notifyWhen value is different from the default value for its summary type - frequency.summary - ? frequency.notifyWhen !== DEFAULT_FREQUENCY_WITH_SUMMARY.notifyWhen - : frequency.notifyWhen !== DEFAULT_FREQUENCY_WITHOUT_SUMMARY.notifyWhen + DEFAULT_FREQUENCY.notifyWhen ); const [summaryMenuOpen, setSummaryMenuOpen] = useState(false); @@ -193,7 +165,6 @@ export const ActionNotifyWhen = ({ (newValue: RuleNotifyWhenType) => { onNotifyWhenChange(newValue); setNotifyWhenValue(newValue); - setNotifyWhenValueChangedFromDefault(true); // Calling onNotifyWhenChange and onThrottleChange at the same time interferes with the React state lifecycle // so wait for onNotifyWhenChange to process before calling onThrottleChange setTimeout( @@ -213,24 +184,10 @@ export const ActionNotifyWhen = ({ onSummaryChange(summary); setSummaryMenuOpen(false); if (summary && frequency.notifyWhen === RuleNotifyWhen.CHANGE) { - // Call onNotifyWhenChange DIRECTLY to bypass setNotifyWhenValueChangedFromDefault onNotifyWhenChange(RuleNotifyWhen.ACTIVE); - // In cases like this: - // 1. User opens form with notifyWhen: THROTTLE - // 2. User sets notifyWhen: CHANGE, notifyWhenValueChangedFromDefault is now true - // 3. User sets summary: true, notifyWhen gets set to CHANGE - // 4. User sets summary: false, notifyWhen should probably get set back to CHANGE - // To make step 4 possible, we have to reset notifyWhenValueChangedFromDefault: - setNotifyWhenValueChangedFromDefault(false); - } else if ( - !summary && - frequency.notifyWhen === RuleNotifyWhen.ACTIVE && - !notifyWhenValueChangedFromDefault - ) { - onNotifyWhenChange(RuleNotifyWhen.CHANGE); } }, - [onSummaryChange, frequency.notifyWhen, onNotifyWhenChange, notifyWhenValueChangedFromDefault] + [onSummaryChange, frequency.notifyWhen, onNotifyWhenChange] ); const summaryOptions = useMemo( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.test.tsx index 445e983d7fb02..73f0810d4f98d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.test.tsx @@ -20,11 +20,9 @@ import { act } from 'react-dom/test-utils'; import { EuiFieldText } from '@elastic/eui'; import { I18nProvider, __IntlProvider as IntlProvider } from '@kbn/i18n-react'; import { render, waitFor, screen } from '@testing-library/react'; -import { - DEFAULT_FREQUENCY_WITHOUT_SUMMARY, - DEFAULT_FREQUENCY_WITH_SUMMARY, -} from '../../../common/constants'; +import { DEFAULT_FREQUENCY } from '../../../common/constants'; import { transformActionVariables } from '../../lib/action_variables'; +import { RuleNotifyWhen } from '@kbn/alerting-plugin/common'; jest.mock('../../../common/lib/kibana'); const actionTypeRegistry = actionTypeRegistryMock.create(); @@ -312,7 +310,7 @@ describe('action_type_form', () => { actionTypeId: '.pagerduty', group: 'default', params: {}, - frequency: DEFAULT_FREQUENCY_WITHOUT_SUMMARY, + frequency: DEFAULT_FREQUENCY, }; const wrapper = render( @@ -320,7 +318,11 @@ describe('action_type_form', () => { index: 1, actionItem, setActionFrequencyProperty: () => { - actionItem.frequency = DEFAULT_FREQUENCY_WITH_SUMMARY; + actionItem.frequency = { + notifyWhen: RuleNotifyWhen.ACTIVE, + throttle: null, + summary: true, + }; }, })} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_reducer.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_reducer.ts index 01f14ba6ff789..55ce45d592d00 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_reducer.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_reducer.ts @@ -10,7 +10,7 @@ import { isEqual } from 'lodash'; import { Reducer } from 'react'; import { RuleActionParam, IntervalSchedule } from '@kbn/alerting-plugin/common'; import { Rule, RuleAction } from '../../../types'; -import { DEFAULT_FREQUENCY_WITHOUT_SUMMARY } from '../../../common/constants'; +import { DEFAULT_FREQUENCY } from '../../../common/constants'; export type InitialRule = Partial & Pick; @@ -201,7 +201,7 @@ export const ruleReducer = ( const updatedAction = { ...oldAction, frequency: { - ...(oldAction.frequency ?? DEFAULT_FREQUENCY_WITHOUT_SUMMARY), + ...(oldAction.frequency ?? DEFAULT_FREQUENCY), [key]: value, }, }; diff --git a/x-pack/plugins/triggers_actions_ui/public/common/constants/action_frequency_types.ts b/x-pack/plugins/triggers_actions_ui/public/common/constants/action_frequency_types.ts index 2aad24fe63846..7ebfc502ed07d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/common/constants/action_frequency_types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/common/constants/action_frequency_types.ts @@ -7,13 +7,7 @@ import { RuleNotifyWhen } from '@kbn/alerting-plugin/common'; -export const DEFAULT_FREQUENCY_WITH_SUMMARY = { - notifyWhen: RuleNotifyWhen.ACTIVE, - throttle: null, - summary: true, -}; - -export const DEFAULT_FREQUENCY_WITHOUT_SUMMARY = { +export const DEFAULT_FREQUENCY = { notifyWhen: RuleNotifyWhen.CHANGE, throttle: null, summary: false, From 01a18df4365d412f5d05f98beafd72c0f3feabe9 Mon Sep 17 00:00:00 2001 From: Jeramy Soucy Date: Mon, 6 Feb 2023 16:31:35 -0500 Subject: [PATCH 020/134] [Docs] Documents constraints of space id in create space API (#150379) closes #150311 Adds wording to clarify that the space ID must be lowercase alphanumeric, but can include underscores and hyphens. Previously this restriction was not documented, but if these requirements are not met the API will respond with a 400. --- docs/api/spaces-management/post.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/spaces-management/post.asciidoc b/docs/api/spaces-management/post.asciidoc index 28d60caa0d333..035fe897da251 100644 --- a/docs/api/spaces-management/post.asciidoc +++ b/docs/api/spaces-management/post.asciidoc @@ -15,7 +15,7 @@ experimental[] Create a {kib} space. ==== Request body `id`:: - (Required, string) The space ID that is part of the Kibana URL when inside the space. You are unable to change the ID with the update operation. + (Required, string) The space ID that is part of the Kibana URL when inside the space. Space IDs are limited to lowercase alphanumeric, underscore, and hyphen characters (a-z, 0-9, '_', and '-'). You are unable to change the ID with the update operation. `name`:: (Required, string) The display name for the space. From 2ff017ed77e474b5ee2c0114dc6e9315423d6134 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 6 Feb 2023 21:41:11 +0000 Subject: [PATCH 021/134] [Security Solution][Alerts] improves IM rule memory usage and performance (#149208) ## Summary - addresses https://github.com/elastic/kibana/issues/148821 - instead of loading all threats result in memory and also creating object map from it(which doubles memory consumption) it creates signals map right straight from threat results(in `getSignalsMatchesFromThreatIndex`) and doesn't keep all threats in memory anymore - because threats are not kept in memory anymore, additional request introduced that fetches threats before enrichments, based on signals map created on the previous stage - [performance measurements](https://github.com/elastic/kibana/issues/148821#issuecomment-1410242932) - additional issues opened while working on current PR: - https://github.com/elastic/kibana/issues/150041 - https://github.com/elastic/kibana/issues/150038 - reduces further number of matched threats in alert to 200, to prevent Kibana browser tab becoming unresponsive ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../threat_mapping/build_threat_enrichment.ts | 29 +- .../threat_mapping/create_event_signal.ts | 50 +- .../enrich_signal_threat_matches.test.ts | 451 ++++-------------- .../enrich_signal_threat_matches.ts | 197 ++------ .../get_signals_map_from_threat_index.mock.ts | 33 ++ .../get_signals_map_from_threat_index.test.ts | 305 ++++++++++++ .../get_signals_map_from_threat_index.ts | 126 +++++ .../signals/threat_mapping/get_threat_list.ts | 24 - .../threat_enrichment_factory.test.ts | 108 +++++ .../threat_enrichment_factory.ts | 76 +++ 10 files changed, 823 insertions(+), 576 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.mock.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts index 32fe9b8dbe8ce..18cf4240d0b18 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts @@ -6,13 +6,11 @@ */ import type { SignalsEnrichment } from '../types'; -import { - enrichSignalThreatMatches, - getSignalMatchesFromThreatList, -} from './enrich_signal_threat_matches'; import type { BuildThreatEnrichmentOptions } from './types'; import { buildThreatMappingFilter } from './build_threat_mapping_filter'; -import { getAllThreatListHits } from './get_threat_list'; +import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index'; + +import { threatEnrichmentFactory } from './threat_enrichment_factory'; // we do want to make extra requests to the threat index to get enrichments from all threats // previously we were enriched alerts only from `currentThreatList` but not all threats @@ -42,7 +40,7 @@ export const buildThreatEnrichment = ({ }, }); - const threatListHits = await getAllThreatListHits({ + const threatSearchParams = { esClient: services.scopedClusterClient.asCurrentUser, threatFilters: [...threatFilters, threatFiltersFromEvents], query: threatQuery, @@ -58,15 +56,20 @@ export const buildThreatEnrichment = ({ runtimeMappings, listClient, exceptionFilter, - }); + }; - const signalMatches = getSignalMatchesFromThreatList(threatListHits); + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams, + eventsCount: signals.length, + }); - return enrichSignalThreatMatches( - signals, - () => Promise.resolve(threatListHits), + const enrichment = threatEnrichmentFactory({ + signalsQueryMap, threatIndicatorPath, - signalMatches - ); + threatFilters, + threatSearchParams, + }); + + return enrichment(signals); }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts index 598730c627185..c006ab528b3c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts @@ -10,18 +10,14 @@ import { getFilter } from '../get_filter'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; import { buildReasonMessageForThreatMatchAlert } from '../reason_formatters'; import type { CreateEventSignalOptions } from './types'; -import type { SearchAfterAndBulkCreateReturnType, SignalSourceHit } from '../types'; -import { getAllThreatListHits } from './get_threat_list'; -import { - enrichSignalThreatMatches, - getSignalMatchesFromThreatList, -} from './enrich_signal_threat_matches'; +import type { SearchAfterAndBulkCreateReturnType } from '../types'; +import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index'; + +import { threatEnrichmentFactory } from './threat_enrichment_factory'; import { getSignalValueMap } from './utils'; export const createEventSignal = async ({ - alertId, bulkCreate, - completeRule, currentResult, currentEventList, eventsTelemetry, @@ -29,7 +25,6 @@ export const createEventSignal = async ({ inputIndex, language, listClient, - outputIndex, query, ruleExecutionLogger, savedId, @@ -69,7 +64,7 @@ export const createEventSignal = async ({ ); return currentResult; } else { - const threatListHits = await getAllThreatListHits({ + const threatSearchParams = { esClient: services.scopedClusterClient.asCurrentUser, threatFilters: [...threatFilters, threatFiltersFromEvents], query: threatQuery, @@ -77,7 +72,7 @@ export const createEventSignal = async ({ index: threatIndex, ruleExecutionLogger, threatListConfig: { - _source: [`${threatIndicatorPath}.*`, 'threat.feed.*', ...threatMatchedFields.threat], + _source: threatMatchedFields.threat, fields: undefined, }, pitId: threatPitId, @@ -85,15 +80,15 @@ export const createEventSignal = async ({ runtimeMappings, listClient, exceptionFilter, - }); - - const signalMatches = getSignalMatchesFromThreatList( - threatListHits, - getSignalValueMap({ eventList: currentEventList, threatMatchedFields }) - ); + }; - const ids = signalMatches.map((item) => item.signalId); + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams, + eventsCount: currentEventList.length, + signalValueMap: getSignalValueMap({ eventList: currentEventList, threatMatchedFields }), + }); + const ids = Array.from(signalsQueryMap.keys()); const indexFilter = { query: { bool: { @@ -115,22 +110,19 @@ export const createEventSignal = async ({ exceptionFilter, }); - ruleExecutionLogger.debug( - `${ids?.length} matched signals found from ${threatListHits.length} indicators` - ); + ruleExecutionLogger.debug(`${ids?.length} matched signals found`); - const threatEnrichment = (signals: SignalSourceHit[]): Promise => - enrichSignalThreatMatches( - signals, - () => Promise.resolve(threatListHits), - threatIndicatorPath, - signalMatches - ); + const enrichment = threatEnrichmentFactory({ + signalsQueryMap, + threatIndicatorPath, + threatFilters, + threatSearchParams, + }); const result = await searchAfterAndBulkCreate({ buildReasonMessage: buildReasonMessageForThreatMatchAlert, bulkCreate, - enrichment: threatEnrichment, + enrichment, eventsTelemetry, exceptionsList: unprocessedExceptions, filter: esFilter, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 974b0e00a7ce4..2658c90720adb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -13,19 +13,11 @@ import type { SignalSourceHit } from '../types'; import { getThreatListItemMock } from './build_threat_mapping_filter.mock'; import { buildEnrichments, - enrichSignalThreatMatches, groupAndMergeSignalMatches, - getSignalMatchesFromThreatList, - MAX_NUMBER_OF_SIGNAL_MATCHES, + enrichSignalThreatMatchesFromSignalsMap, } from './enrich_signal_threat_matches'; import { getNamedQueryMock, getSignalHitMock } from './enrich_signal_threat_matches.mock'; -import type { - GetMatchedThreats, - ThreatListItem, - ThreatMatchNamedQuery, - SignalMatch, -} from './types'; -import { encodeThreatMatchNamedQuery } from './utils'; +import type { ThreatListItem, ThreatMatchNamedQuery } from './types'; describe('groupAndMergeSignalMatches', () => { it('returns an empty array if there are no signals', () => { @@ -481,11 +473,10 @@ describe('buildEnrichments', () => { }); }); -describe('enrichSignalThreatMatches', () => { - let getMatchedThreats: GetMatchedThreats; - let matchedQuery: string; +describe('enrichSignalThreatMatchesFromSignalsMap', () => { + let getMatchedThreats: () => Promise; let indicatorPath: string; - let signalMatches: SignalMatch[]; + let signalsMap = new Map(); beforeEach(() => { indicatorPath = 'threat.indicator'; @@ -500,56 +491,67 @@ describe('enrichSignalThreatMatches', () => { }, }), ]; - matchedQuery = encodeThreatMatchNamedQuery( - getNamedQueryMock({ - id: '123', - index: 'indicator_index', - field: 'event.domain', - value: 'threat.indicator.domain', - }) - ); - signalMatches = [ - { - signalId: '_id', - queries: [ - getNamedQueryMock({ + signalsMap = new Map([ + [ + 'source-id', + [ + { id: '123', index: 'indicator_index', field: 'event.domain', value: 'threat.indicator.domain', - }), + }, ], - }, - ]; + ], + ]); }); it('performs no enrichment if there are no signals', async () => { const signals: SignalSourceHit[] = []; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, indicatorPath, - [] + new Map() ); expect(enrichedSignals).toEqual([]); }); + it('performs no enrichment if signalsMap empty', async () => { + const signalHit = getSignalHitMock({ + _source: { + '@timestamp': 'mocked', + event: { category: 'malware', domain: 'domain_1' }, + threat: { enrichments: [{ existing: 'indicator' }] }, + }, + }); + const signals: SignalSourceHit[] = [signalHit]; + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( + signals, + getMatchedThreats, + indicatorPath, + new Map() + ); + + expect(enrichedSignals).toEqual([signalHit]); + }); + it('preserves existing threat.enrichments objects on signals', async () => { const signalHit = getSignalHitMock({ + _id: 'source-id', _source: { '@timestamp': 'mocked', event: { category: 'malware', domain: 'domain_1' }, threat: { enrichments: [{ existing: 'indicator' }] }, }, - matched_queries: [matchedQuery], }); const signals: SignalSourceHit[] = [signalHit]; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, indicatorPath, - signalMatches + signalsMap ); const [enrichedHit] = enrichedSignals; const enrichments = get(enrichedHit._source, ENRICHMENT_DESTINATION_PATH); @@ -577,14 +579,14 @@ describe('enrichSignalThreatMatches', () => { it('provides only match data if the matched threat cannot be found', async () => { getMatchedThreats = async () => []; const signalHit = getSignalHitMock({ - matched_queries: [matchedQuery], + _id: 'source-id', }); const signals: SignalSourceHit[] = [signalHit]; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, indicatorPath, - signalMatches + signalsMap ); const [enrichedHit] = enrichedSignals; const enrichments = get(enrichedHit._source, ENRICHMENT_DESTINATION_PATH); @@ -606,6 +608,7 @@ describe('enrichSignalThreatMatches', () => { it('preserves an existing threat.enrichments object on signals', async () => { const signalHit = getSignalHitMock({ + _id: 'source-id', _source: { '@timestamp': 'mocked', event: { category: 'virus', domain: 'domain_1' }, @@ -616,14 +619,13 @@ describe('enrichSignalThreatMatches', () => { ], }, }, - matched_queries: [matchedQuery], }); const signals: SignalSourceHit[] = [signalHit]; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, indicatorPath, - signalMatches + signalsMap ); const [enrichedHit] = enrichedSignals; const enrichments = get(enrichedHit._source, ENRICHMENT_DESTINATION_PATH); @@ -656,15 +658,28 @@ describe('enrichSignalThreatMatches', () => { it('throws an error if threat is neither an object nor undefined', async () => { const signalHit = getSignalHitMock({ _source: { '@timestamp': 'mocked', threat: 'whoops' }, - matched_queries: [matchedQuery], }); const signals: SignalSourceHit[] = [signalHit]; await expect(() => - enrichSignalThreatMatches(signals, getMatchedThreats, indicatorPath, signalMatches) + enrichSignalThreatMatchesFromSignalsMap(signals, getMatchedThreats, indicatorPath, signalsMap) ).rejects.toThrowError('Expected threat field to be an object, but found: whoops'); }); it('enriches from a configured indicator path, if specified', async () => { + signalsMap = new Map([ + [ + 'source-id', + [ + { + id: '123', + index: 'custom_index', + field: 'event.domain', + value: 'threat.indicator.domain', + }, + ], + ], + ]); + getMatchedThreats = async () => [ getThreatListItemMock({ _id: '123', @@ -679,27 +694,20 @@ describe('enrichSignalThreatMatches', () => { }, }), ]; - const namedQuery = getNamedQueryMock({ - id: '123', - index: 'custom_index', - field: 'event.domain', - value: 'custom_threat.custom_indicator.domain', - }); - matchedQuery = encodeThreatMatchNamedQuery(namedQuery); const signalHit = getSignalHitMock({ + _id: 'source-id', _source: { event: { domain: 'domain_1', }, }, - matched_queries: [matchedQuery], }); const signals: SignalSourceHit[] = [signalHit]; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, 'custom_threat.custom_indicator', - [{ signalId: '_id', queries: [namedQuery] }] + signalsMap ); const [enrichedHit] = enrichedSignals; const enrichments = get(enrichedHit._source, ENRICHMENT_DESTINATION_PATH); @@ -724,6 +732,26 @@ describe('enrichSignalThreatMatches', () => { }); it('merges duplicate matched signals into a single signal with multiple enrichments', async () => { + signalsMap = new Map([ + [ + 'source-id', + [ + { + id: '123', + index: 'indicator_index', + field: 'event.domain', + value: 'threat.indicator.domain', + }, + { + id: '456', + index: 'other_custom_index', + field: 'event.other', + value: 'threat.indicator.domain', + }, + ], + ], + ]); + getMatchedThreats = async () => [ getThreatListItemMock({ _id: '123', @@ -741,50 +769,29 @@ describe('enrichSignalThreatMatches', () => { }), ]; const signalHit = getSignalHitMock({ - _id: 'signal123', + _id: 'source-id', _source: { event: { domain: 'domain_1', other: 'test_val', }, }, - matched_queries: [matchedQuery], - }); - const otherMatchQuery = getNamedQueryMock({ - id: '456', - index: 'other_custom_index', - field: 'event.other', - value: 'threat.indicator.domain', }); const otherSignalHit = getSignalHitMock({ - _id: 'signal123', + _id: 'source-id', _source: { event: { domain: 'domain_1', other: 'test_val', }, }, - matched_queries: [encodeThreatMatchNamedQuery(otherMatchQuery)], }); const signals: SignalSourceHit[] = [signalHit, otherSignalHit]; - const enrichedSignals = await enrichSignalThreatMatches( + const enrichedSignals = await enrichSignalThreatMatchesFromSignalsMap( signals, getMatchedThreats, indicatorPath, - [ - { - signalId: 'signal123', - queries: [ - getNamedQueryMock({ - id: '123', - index: 'indicator_index', - field: 'event.domain', - value: 'threat.indicator.domain', - }), - otherMatchQuery, - ], - }, - ] + signalsMap ); expect(enrichedSignals).toHaveLength(1); @@ -825,291 +832,3 @@ describe('enrichSignalThreatMatches', () => { ]); }); }); - -describe('getSignalMatchesFromThreatList', () => { - it('return empty array if there no threat indicators', () => { - const signalMatches = getSignalMatchesFromThreatList(); - expect(signalMatches).toEqual([]); - }); - - it("return empty array if threat indicators doesn't have matched query", () => { - const signalMatches = getSignalMatchesFromThreatList([getThreatListItemMock()]); - expect(signalMatches).toEqual([]); - }); - - it('return signal matches from threat indicators', () => { - const signalMatches = getSignalMatchesFromThreatList([ - getThreatListItemMock({ - _id: 'threatId', - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - id: 'signalId1', - index: 'source_index', - value: 'threat.indicator.domain', - field: 'event.domain', - }) - ), - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - id: 'signalId2', - index: 'source_index', - value: 'threat.indicator.domain', - field: 'event.domain', - }) - ), - ], - }), - ]); - - const queries = [ - { - field: 'event.domain', - value: 'threat.indicator.domain', - index: 'threat_index', - id: 'threatId', - queryType: 'mq', - }, - ]; - - expect(signalMatches).toEqual([ - { - signalId: 'signalId1', - queries, - }, - { - signalId: 'signalId2', - queries, - }, - ]); - }); - - it('return empty array for terms query if there no signalValueMap', () => { - const signalMatches = getSignalMatchesFromThreatList([ - getThreatListItemMock({ - _id: 'threatId', - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - value: 'threat.indicator.domain', - field: 'event.domain', - queryType: 'tq', - }) - ), - ], - }), - ]); - - expect(signalMatches).toEqual([]); - }); - - it('return empty array for terms query if there wrong value in threat indicator', () => { - const threat = getThreatListItemMock({ - _id: 'threatId', - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - value: 'threat.indicator.domain', - field: 'event.domain', - queryType: 'tq', - }) - ), - ], - }); - - threat._source = { - ...threat._source, - threat: { - indicator: { - domain: { a: 'b' }, - }, - }, - }; - - const signalValueMap = { - 'event.domain': { - domain_1: ['signalId1', 'signalId2'], - }, - }; - - const signalMatches = getSignalMatchesFromThreatList([threat], signalValueMap); - - expect(signalMatches).toEqual([]); - }); - - it('return signal matches from threat indicators for termsQuery', () => { - const threat = getThreatListItemMock({ - _id: 'threatId', - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - value: 'threat.indicator.domain', - field: 'event.domain', - queryType: 'tq', - }) - ), - ], - }); - - threat._source = { - ...threat._source, - threat: { - indicator: { - domain: 'domain_1', - }, - }, - }; - - const signalValueMap = { - 'event.domain': { - domain_1: ['signalId1', 'signalId2'], - }, - }; - - const signalMatches = getSignalMatchesFromThreatList([threat], signalValueMap); - - const queries = [ - { - field: 'event.domain', - value: 'threat.indicator.domain', - index: 'threat_index', - id: 'threatId', - queryType: 'tq', - }, - ]; - - expect(signalMatches).toEqual([ - { - signalId: 'signalId1', - queries, - }, - { - signalId: 'signalId2', - queries, - }, - ]); - }); - - it('return signal matches from threat indicators which has array values for termsQuery', () => { - const threat = getThreatListItemMock({ - _id: 'threatId', - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - value: 'threat.indicator.domain', - field: 'event.domain', - queryType: 'tq', - }) - ), - ], - }); - - threat._source = { - ...threat._source, - threat: { - indicator: { - domain: ['domain_3', 'domain_1', 'domain_2'], - }, - }, - }; - - const signalValueMap = { - 'event.domain': { - domain_1: ['signalId1'], - domain_2: ['signalId2'], - }, - }; - - const signalMatches = getSignalMatchesFromThreatList([threat], signalValueMap); - - const queries = [ - { - field: 'event.domain', - value: 'threat.indicator.domain', - index: 'threat_index', - id: 'threatId', - queryType: 'tq', - }, - ]; - - expect(signalMatches).toEqual([ - { - signalId: 'signalId1', - queries, - }, - { - signalId: 'signalId2', - queries, - }, - ]); - }); - - it('merge signal matches if different threat indicators matched the same signal', () => { - const matchedQuery = [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - id: 'signalId', - index: 'source_index', - value: 'threat.indicator.domain', - field: 'event.domain', - }) - ), - ]; - const signalMatches = getSignalMatchesFromThreatList([ - getThreatListItemMock({ - _id: 'threatId1', - matched_queries: matchedQuery, - }), - getThreatListItemMock({ - _id: 'threatId2', - matched_queries: matchedQuery, - }), - ]); - - const query = { - field: 'event.domain', - value: 'threat.indicator.domain', - index: 'threat_index', - id: 'threatId', - queryType: 'mq', - }; - - expect(signalMatches).toEqual([ - { - signalId: 'signalId', - queries: [ - { - ...query, - id: 'threatId1', - }, - { - ...query, - id: 'threatId2', - }, - ], - }, - ]); - }); - - it('limits number of signal matches to MAX_NUMBER_OF_SIGNAL_MATCHES', () => { - const threatList = Array.from(Array(2000), (index) => - getThreatListItemMock({ - _id: `threatId-${index}`, - matched_queries: [ - encodeThreatMatchNamedQuery( - getNamedQueryMock({ - id: 'signalId1', - index: 'source_index', - value: 'threat.indicator.domain', - field: 'event.domain', - }) - ), - ], - }) - ); - - const signalMatches = getSignalMatchesFromThreatList(threatList); - - expect(signalMatches[0].queries).toHaveLength(MAX_NUMBER_OF_SIGNAL_MATCHES); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 3ce871283219d..28f43519f118f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -9,102 +9,9 @@ import { get, isObject } from 'lodash'; import { ENRICHMENT_TYPES, FEED_NAME_PATH } from '../../../../../common/cti/constants'; import type { SignalSourceHit } from '../types'; -import type { - GetMatchedThreats, - ThreatEnrichment, - ThreatListItem, - ThreatMatchNamedQuery, - SignalMatch, - SignalValuesMap, - ThreatTermNamedQuery, -} from './types'; -import { ThreatMatchQueryType } from './types'; -import { extractNamedQueries } from './utils'; - -export const MAX_NUMBER_OF_SIGNAL_MATCHES = 1000; - -export const getSignalMatchesFromThreatList = ( - threatList: ThreatListItem[] = [], - signalValueMap?: SignalValuesMap -): SignalMatch[] => { - const signalMap: { [key: string]: ThreatMatchNamedQuery[] } = {}; - const addSignalValueToMap = ({ - id, - threatHit, - query, - }: { - id: string; - threatHit: ThreatListItem; - query: ThreatMatchNamedQuery | ThreatTermNamedQuery; - }) => { - if (!signalMap[id]) { - signalMap[id] = []; - } - - // creating map of signal with large number of threats could lead to out of memory Kibana crash - // large number of threats also can cause signals bulk create failure due too large payload (413) - // large number of threats significantly slower alert details page render - // so, its number is limited to MAX_NUMBER_OF_SIGNAL_MATCHES - // more details https://github.com/elastic/kibana/issues/143595#issuecomment-1335433592 - if (signalMap[id].length >= MAX_NUMBER_OF_SIGNAL_MATCHES) { - return; - } - - signalMap[id].push({ - id: threatHit._id, - index: threatHit._index, - field: query.field, - value: query.value, - queryType: query.queryType, - }); - }; - threatList.forEach((threatHit) => - extractNamedQueries(threatHit).forEach((query) => { - const signalId = query.id; - - if (query.queryType === ThreatMatchQueryType.term) { - const threatValue = get(threatHit?._source, query.value); - let values; - if (Array.isArray(threatValue)) { - values = threatValue; - } else { - values = [threatValue]; - } - - values.forEach((value) => { - if (value && signalValueMap) { - const ids = signalValueMap[query.field][value?.toString()]; - - ids?.forEach((id: string) => { - addSignalValueToMap({ - id, - threatHit, - query, - }); - }); - } - }); - } else { - if (!signalId) { - return; - } - - addSignalValueToMap({ - id: signalId, - threatHit, - query, - }); - } - }) - ); +import type { ThreatEnrichment, ThreatListItem, ThreatMatchNamedQuery } from './types'; - const signalMatches = Object.entries(signalMap).map(([key, value]) => ({ - signalId: key, - queries: value, - })); - - return signalMatches; -}; +export const MAX_NUMBER_OF_SIGNAL_MATCHES = 200; const getSignalId = (signal: SignalSourceHit): string => signal._id; @@ -163,68 +70,70 @@ export const buildEnrichments = ({ }; }); -export const enrichSignalThreatMatches = async ( +const enrichSignalWithThreatMatches = ( + signalHit: SignalSourceHit, + enrichmentsWithoutAtomic: { [key: string]: ThreatEnrichment[] } +) => { + const threat = get(signalHit._source, 'threat') ?? {}; + if (!isObject(threat)) { + throw new Error(`Expected threat field to be an object, but found: ${threat}`); + } + // We are not using ENRICHMENT_DESTINATION_PATH here because the code above + // and below make assumptions about its current value, 'threat.enrichments', + // and making this code dynamic on an arbitrary path would introduce several + // new issues. + const existingEnrichmentValue = get(signalHit._source, 'threat.enrichments') ?? []; + const existingEnrichments = [existingEnrichmentValue].flat(); // ensure enrichments is an array + const newEnrichmentsWithoutAtomic = enrichmentsWithoutAtomic[signalHit._id] ?? []; + const newEnrichments = newEnrichmentsWithoutAtomic.map((enrichment) => ({ + ...enrichment, + matched: { + ...enrichment.matched, + atomic: get(signalHit._source, enrichment.matched.field), + }, + })); + + return { + ...signalHit, + _source: { + ...signalHit._source, + threat: { + ...threat, + enrichments: [...existingEnrichments, ...newEnrichments], + }, + }, + }; +}; + +/** + * enrich signals threat matches using signalsMap(Map) that has match named query results + */ +export const enrichSignalThreatMatchesFromSignalsMap = async ( signals: SignalSourceHit[], - getMatchedThreats: GetMatchedThreats, + getMatchedThreats: () => Promise, indicatorPath: string, - signalMatches: SignalMatch[] + signalsMap: Map ): Promise => { if (signals.length === 0) { - return signals; + return []; } const uniqueHits = groupAndMergeSignalMatches(signals); + const matchedThreats = await getMatchedThreats(); - const matchedThreatIds = [ - ...new Set( - signalMatches - .map((signalMatch) => signalMatch.queries) - .flat() - .map(({ id }) => id) - ), - ]; - const matchedThreats = await getMatchedThreats(matchedThreatIds); - - const enrichmentsWithoutAtomic: { [key: string]: ThreatEnrichment[] } = {}; - signalMatches.forEach((signalMatch) => { - enrichmentsWithoutAtomic[signalMatch.signalId] = buildEnrichments({ + const enrichmentsWithoutAtomic: Record = {}; + + uniqueHits.forEach((hit) => { + enrichmentsWithoutAtomic[hit._id] = buildEnrichments({ indicatorPath, - queries: signalMatch.queries, + queries: signalsMap.get(hit._id) ?? [], threats: matchedThreats, }); }); - const enrichedSignals: SignalSourceHit[] = uniqueHits.map((signalHit, i) => { - const threat = get(signalHit._source, 'threat') ?? {}; - if (!isObject(threat)) { - throw new Error(`Expected threat field to be an object, but found: ${threat}`); - } - // We are not using ENRICHMENT_DESTINATION_PATH here because the code above - // and below make assumptions about its current value, 'threat.enrichments', - // and making this code dynamic on an arbitrary path would introduce several - // new issues. - const existingEnrichmentValue = get(signalHit._source, 'threat.enrichments') ?? []; - const existingEnrichments = [existingEnrichmentValue].flat(); // ensure enrichments is an array - const newEnrichmentsWithoutAtomic = enrichmentsWithoutAtomic[signalHit._id] ?? []; - const newEnrichments = newEnrichmentsWithoutAtomic.map((enrichment) => ({ - ...enrichment, - matched: { - ...enrichment.matched, - atomic: get(signalHit._source, enrichment.matched.field), - }, - })); - - return { - ...signalHit, - _source: { - ...signalHit._source, - threat: { - ...threat, - enrichments: [...existingEnrichments, ...newEnrichments], - }, - }, - }; - }); + const enrichedSignals: SignalSourceHit[] = uniqueHits.map((signalHit) => + enrichSignalWithThreatMatches(signalHit, enrichmentsWithoutAtomic) + ); return enrichedSignals; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.mock.ts new file mode 100644 index 0000000000000..9fa23d25f2899 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.mock.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import type { GetThreatListOptions } from './types'; +import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; + +const esClient = elasticsearchServiceMock.createElasticsearchClient(); +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +export const threatSearchParamsMock: GetThreatListOptions = { + esClient, + query: '*:*', + language: 'kuery', + threatFilters: [], + index: ['threats-*'], + ruleExecutionLogger, + threatListConfig: { + _source: false, + fields: undefined, + }, + pitId: 'mock', + reassignPitId: jest.fn(), + listClient: getListClientMock(), + searchAfter: undefined, + runtimeMappings: undefined, + exceptionFilter: undefined, +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts new file mode 100644 index 0000000000000..38a6947beebcb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.test.ts @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ThreatMatchQueryType } from './types'; + +import { getSignalsQueryMapFromThreatIndex } from './get_signals_map_from_threat_index'; +import { getThreatList } from './get_threat_list'; +import { encodeThreatMatchNamedQuery } from './utils'; +import { MAX_NUMBER_OF_SIGNAL_MATCHES } from './enrich_signal_threat_matches'; + +import { threatSearchParamsMock } from './get_signals_map_from_threat_index.mock'; + +jest.mock('./get_threat_list', () => ({ getThreatList: jest.fn() })); + +const getThreatListMock = getThreatList as jest.Mock; + +export const namedQuery = encodeThreatMatchNamedQuery({ + id: 'source-1', + index: 'source-*', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, +}); + +const termsNamedQuery = encodeThreatMatchNamedQuery({ + value: 'threat.indicator.domain', + field: 'event.domain', + queryType: ThreatMatchQueryType.term, +}); + +export const threatMock = { + _id: 'threat-id-1', + _index: 'threats-01', + matched_queries: [namedQuery], +}; + +const termsThreatMock = { + _id: 'threat-id-1', + _index: 'threats-01', + matched_queries: [termsNamedQuery], +}; + +getThreatListMock.mockReturnValue({ hits: { hits: [] } }); + +describe('getSignalsQueryMapFromThreatIndex', () => { + it('should call getThreatList to fetch threats from ES', async () => { + getThreatListMock.mockReturnValue({ hits: { hits: [] } }); + + await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + }); + + expect(getThreatListMock).toHaveBeenCalledTimes(1); + expect(getThreatListMock).toHaveBeenCalledWith(threatSearchParamsMock); + }); + + it('should return empty signals map if getThreatList return empty results', async () => { + getThreatListMock.mockReturnValue({ hits: { hits: [] } }); + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + }); + + expect(signalsQueryMap).toEqual(new Map()); + }); + + it('should return signalsQueryMap for signals if threats search results exhausted', async () => { + const namedQuery2 = encodeThreatMatchNamedQuery({ + id: 'source-2', + index: 'source-*', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, + }); + + // the third request return empty results + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [threatMock], + }, + }); + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [ + { ...threatMock, _id: 'threat-id-2', matched_queries: [namedQuery, namedQuery2] }, + { ...threatMock, _id: 'threat-id-3' }, + ], + }, + }); + getThreatListMock.mockReturnValueOnce({ hits: { hits: [] } }); + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + }); + + expect(signalsQueryMap).toEqual( + new Map([ + [ + 'source-1', + [ + { + id: 'threat-id-1', + index: 'threats-01', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, + }, + { + id: 'threat-id-2', + index: 'threats-01', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, + }, + { + id: 'threat-id-3', + index: 'threats-01', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, + }, + ], + ], + [ + 'source-2', + [ + { + id: 'threat-id-2', + index: 'threats-01', + field: 'host.name', + value: 'localhost-1', + queryType: ThreatMatchQueryType.match, + }, + ], + ], + ]) + ); + }); + it('should return signalsQueryMap for signals if threats number reaches max of MAX_NUMBER_OF_SIGNAL_MATCHES', async () => { + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: Array.from(Array(MAX_NUMBER_OF_SIGNAL_MATCHES + 1)).map(() => threatMock), + }, + }); + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + }); + + expect(signalsQueryMap.get('source-1')).toHaveLength(MAX_NUMBER_OF_SIGNAL_MATCHES); + }); + + it('should return empty signalsQueryMap for terms query if there no signalValueMap', async () => { + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [termsThreatMock], + }, + }); + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + }); + + expect(signalsQueryMap).toEqual(new Map()); + }); + + it('should return empty signalsQueryMap for terms query if there wrong value in threat indicator', async () => { + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [ + { + ...termsThreatMock, + _source: { + threat: { + indicator: { + domain: { a: 'b' }, + }, + }, + }, + }, + ], + }, + }); + + const signalValueMap = { + 'event.domain': { + domain_1: ['signalId1', 'signalId2'], + }, + }; + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + signalValueMap, + }); + + expect(signalsQueryMap).toEqual(new Map()); + }); + + it('should return signalsQueryMap from threat indicators for termsQuery', async () => { + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [ + { + ...termsThreatMock, + _source: { + threat: { + indicator: { + domain: 'domain_1', + }, + }, + }, + }, + ], + }, + }); + + const signalValueMap = { + 'event.domain': { + domain_1: ['signalId1', 'signalId2'], + }, + }; + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + signalValueMap, + }); + + const queries = [ + { + field: 'event.domain', + value: 'threat.indicator.domain', + id: 'threat-id-1', + index: 'threats-01', + queryType: ThreatMatchQueryType.term, + }, + ]; + + expect(signalsQueryMap).toEqual( + new Map([ + ['signalId1', queries], + ['signalId2', queries], + ]) + ); + }); + + it('should return signalsQueryMap from threat indicators which has array values for termsQuery', async () => { + getThreatListMock.mockReturnValueOnce({ + hits: { + hits: [ + { + ...termsThreatMock, + _source: { + threat: { + indicator: { + domain: ['domain_3', 'domain_1', 'domain_2'], + }, + }, + }, + }, + ], + }, + }); + + const signalValueMap = { + 'event.domain': { + domain_1: ['signalId1'], + domain_2: ['signalId2'], + }, + }; + + const signalsQueryMap = await getSignalsQueryMapFromThreatIndex({ + threatSearchParams: threatSearchParamsMock, + eventsCount: 50, + signalValueMap, + }); + + const queries = [ + { + field: 'event.domain', + value: 'threat.indicator.domain', + id: 'threat-id-1', + index: 'threats-01', + queryType: ThreatMatchQueryType.term, + }, + ]; + + expect(signalsQueryMap).toEqual( + new Map([ + ['signalId1', queries], + ['signalId2', queries], + ]) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts new file mode 100644 index 0000000000000..0deb3beeee2e8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_signals_map_from_threat_index.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { get } from 'lodash'; + +import { ThreatMatchQueryType } from './types'; +import type { + GetThreatListOptions, + ThreatMatchNamedQuery, + ThreatTermNamedQuery, + ThreatListItem, + SignalValuesMap, +} from './types'; +import { getThreatList } from './get_threat_list'; +import { decodeThreatMatchNamedQuery } from './utils'; + +import { MAX_NUMBER_OF_SIGNAL_MATCHES } from './enrich_signal_threat_matches'; + +export type SignalsQueryMap = Map; + +interface GetSignalsMatchesFromThreatIndexOptions { + threatSearchParams: Omit; + eventsCount: number; + signalValueMap?: SignalValuesMap; +} + +/** + * fetches threats and creates signals map from results, that matches signal is with list of threat queries + */ +export const getSignalsQueryMapFromThreatIndex = async ({ + threatSearchParams, + eventsCount, + signalValueMap, +}: GetSignalsMatchesFromThreatIndexOptions): Promise => { + let threatList: Awaited> | undefined; + const signalsQueryMap = new Map(); + // number of threat matches per signal is limited by MAX_NUMBER_OF_SIGNAL_MATCHES. Once it hits this number, threats stop to be processed for a signal + const maxThreatsReachedMap = new Map(); + + const addSignalValueToMap = ({ + signalId, + threatHit, + decodedQuery, + }: { + signalId: string; + threatHit: ThreatListItem; + decodedQuery: ThreatMatchNamedQuery | ThreatTermNamedQuery; + }) => { + const signalMatch = signalsQueryMap.get(signalId); + if (!signalMatch) { + signalsQueryMap.set(signalId, []); + } + + const threatQuery = { + id: threatHit._id, + index: threatHit._index, + field: decodedQuery.field, + value: decodedQuery.value, + queryType: decodedQuery.queryType, + }; + + if (!signalMatch) { + signalsQueryMap.set(signalId, [threatQuery]); + return; + } + + if (signalMatch.length === MAX_NUMBER_OF_SIGNAL_MATCHES) { + maxThreatsReachedMap.set(signalId, true); + } else if (signalMatch.length < MAX_NUMBER_OF_SIGNAL_MATCHES) { + signalMatch.push(threatQuery); + } + }; + + while ( + maxThreatsReachedMap.size < eventsCount && + (threatList ? threatList?.hits.hits.length > 0 : true) + ) { + threatList = await getThreatList({ + ...threatSearchParams, + searchAfter: threatList?.hits.hits[threatList.hits.hits.length - 1].sort || undefined, + }); + + threatList.hits.hits.forEach((threatHit) => { + const matchedQueries = threatHit?.matched_queries || []; + + matchedQueries.forEach((matchedQuery) => { + const decodedQuery = decodeThreatMatchNamedQuery(matchedQuery); + const signalId = decodedQuery.id; + + if (decodedQuery.queryType === ThreatMatchQueryType.term) { + const threatValue = get(threatHit?._source, decodedQuery.value); + const values = Array.isArray(threatValue) ? threatValue : [threatValue]; + + values.forEach((value) => { + if (value && signalValueMap) { + const ids = signalValueMap[decodedQuery.field][value?.toString()]; + + ids?.forEach((id: string) => { + addSignalValueToMap({ + signalId: id, + threatHit, + decodedQuery, + }); + }); + } + }); + } else { + if (!signalId) { + return; + } + + addSignalValueToMap({ + signalId, + threatHit, + decodedQuery, + }); + } + }); + }); + } + + return signalsQueryMap; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index c2ee1fee3c75a..160de278f47e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -11,7 +11,6 @@ import type { GetThreatListOptions, ThreatListCountOptions, ThreatListDoc, - ThreatListItem, GetSortForThreatList, } from './types'; @@ -20,8 +19,6 @@ import type { */ export const INDICATOR_PER_PAGE = 1000; -const MAX_NUMBER_OF_THREATS = 10 * 1000; - export const getThreatList = async ({ esClient, index, @@ -116,24 +113,3 @@ export const getThreatListCount = async ({ }); return response.count; }; - -export const getAllThreatListHits = async ( - params: Omit -): Promise => { - let allThreatListHits: ThreatListItem[] = []; - let threatList = await getThreatList({ ...params, searchAfter: undefined }); - - allThreatListHits = allThreatListHits.concat(threatList.hits.hits); - - // to prevent loading in memory large number of results, that could lead to out of memory Kibana crash, - // number of indicators is limited to MAX_NUMBER_OF_THREATS - while (threatList.hits.hits.length !== 0 && allThreatListHits.length < MAX_NUMBER_OF_THREATS) { - threatList = await getThreatList({ - ...params, - searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort, - }); - - allThreatListHits = allThreatListHits.concat(threatList.hits.hits); - } - return allThreatListHits; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.test.ts new file mode 100644 index 0000000000000..3c6bb7d8ee169 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getThreatList } from './get_threat_list'; +import { getNamedQueryMock } from './enrich_signal_threat_matches.mock'; +import type { SignalSourceHit } from '../types'; +import { threatSearchParamsMock } from './get_signals_map_from_threat_index.mock'; +import { threatEnrichmentFactory } from './threat_enrichment_factory'; +import { enrichSignalThreatMatchesFromSignalsMap } from './enrich_signal_threat_matches'; + +jest.mock('./get_threat_list', () => ({ getThreatList: jest.fn() })); +jest.mock('./enrich_signal_threat_matches', () => ({ + enrichSignalThreatMatchesFromSignalsMap: jest.fn(), +})); + +const getThreatListMock = getThreatList as jest.Mock; +const enrichSignalThreatMatchesFromSignalsMapMock = + enrichSignalThreatMatchesFromSignalsMap as jest.Mock; + +const signals = [ + { + _id: 'source-id-1', + }, + { + _id: 'source-id-2', + }, +]; + +const signalsMapMock = new Map([ + ['source-id-1', [getNamedQueryMock({ id: 'threat-1' }), getNamedQueryMock({ id: 'threat-2' })]], + // this signal's threats not present in signalsMap, so will be ignored in threatFilter + ['source-id-3', [getNamedQueryMock({ id: 'threat-x' }), getNamedQueryMock({ id: 'threat-y' })]], +]); + +getThreatListMock.mockReturnValue({ hits: { hits: [] } }); +enrichSignalThreatMatchesFromSignalsMapMock.mockImplementation((_, getThreats) => getThreats()); + +describe('threatEnrichmentFactory', () => { + it('enrichment should call enrichSignalThreatMatchesFromSignalsMap with correct params', async () => { + const enrichment = threatEnrichmentFactory({ + signalsQueryMap: signalsMapMock, + threatIndicatorPath: 'indicator.mock', + threatFilters: ['mock-threat-filters'], + threatSearchParams: threatSearchParamsMock, + }); + + await enrichment(signals as SignalSourceHit[]); + + expect(enrichSignalThreatMatchesFromSignalsMap).toHaveBeenCalledWith( + signals, + expect.any(Function), + 'indicator.mock', + signalsMapMock + ); + }); + + it('enrichment should call getThreatList with matched threat ids filters in signalsMap', async () => { + const enrichment = threatEnrichmentFactory({ + signalsQueryMap: signalsMapMock, + threatIndicatorPath: 'indicator.mock', + threatFilters: ['mock-threat-filters'], + threatSearchParams: threatSearchParamsMock, + }); + + await enrichment(signals as SignalSourceHit[]); + + expect(getThreatListMock).toHaveBeenCalledWith( + expect.objectContaining({ + threatFilters: [ + 'mock-threat-filters', + { + query: { + bool: { + filter: { + ids: { values: ['threat-1', 'threat-2'] }, + }, + }, + }, + }, + ], + }) + ); + }); + + it('enrichment should call getThreatList with correct threatListConfig', async () => { + const enrichment = threatEnrichmentFactory({ + signalsQueryMap: new Map(), + threatIndicatorPath: 'indicator.mock', + threatFilters: ['mock-threat-filters'], + threatSearchParams: threatSearchParamsMock, + }); + + await enrichment(signals as SignalSourceHit[]); + + expect(getThreatListMock).toHaveBeenCalledWith( + expect.objectContaining({ + threatListConfig: { + _source: ['indicator.mock.*', 'threat.feed.*'], + fields: undefined, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.ts new file mode 100644 index 0000000000000..9d8da19f613bb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/threat_enrichment_factory.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CreateEventSignalOptions, GetThreatListOptions } from './types'; +import type { SignalSourceHit } from '../types'; +import { getThreatList } from './get_threat_list'; +import { enrichSignalThreatMatchesFromSignalsMap } from './enrich_signal_threat_matches'; +import { type SignalsQueryMap } from './get_signals_map_from_threat_index'; + +interface ThreatEnrichmentFactoryOptions { + threatIndicatorPath: CreateEventSignalOptions['threatIndicatorPath']; + signalsQueryMap: SignalsQueryMap; + threatFilters: CreateEventSignalOptions['threatFilters']; + threatSearchParams: Omit; +} + +/** + * returns threatEnrichment method used events-first search + */ +export const threatEnrichmentFactory = ({ + signalsQueryMap, + threatIndicatorPath, + threatFilters, + threatSearchParams, +}: ThreatEnrichmentFactoryOptions) => { + const threatEnrichment = (signals: SignalSourceHit[]): Promise => { + const getThreats = async () => { + const threatIds = signals + .map((s) => s._id) + .reduce((acc, id) => { + return [ + ...new Set([ + ...acc, + ...(signalsQueryMap.get(id) ?? []).map((threatQueryMatched) => threatQueryMatched.id), + ]), + ]; + }, []) + .flat(); + + const matchedThreatsFilter = { + query: { + bool: { + filter: { + ids: { values: threatIds }, + }, + }, + }, + }; + + const threatResponse = await getThreatList({ + ...threatSearchParams, + threatListConfig: { + _source: [`${threatIndicatorPath}.*`, 'threat.feed.*'], + fields: undefined, + }, + threatFilters: [...threatFilters, matchedThreatsFilter], + searchAfter: undefined, + }); + + return threatResponse.hits.hits; + }; + + return enrichSignalThreatMatchesFromSignalsMap( + signals, + getThreats, + threatIndicatorPath, + signalsQueryMap + ); + }; + + return threatEnrichment; +}; From 79a8c2db63ebb87a20b4442f6023b7a500cad16d Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 6 Feb 2023 16:46:16 -0500 Subject: [PATCH 022/134] [Fleet] openapi spec - remove duplicated path parameters (#150260) Several path parameter definitions were duplicated between the path's `parameters` and the method's `parameters`. These two lists are joined so there is no need to re-declare a parameter. I observed these errors while using github.com/deepmap/oapi-codegen with the Fleet openapi definition. ``` path '/package_policies/{packagePolicyId}' has 1 positional parameters, but spec has 2 declared path '/epm/packages/{pkgName}/{pkgVersion}' has 2 positional parameters, but spec has 4 declared path '/fleet_server_hosts/{itemId}' has 1 positional parameters, but spec has 2 declared path '/proxies/{itemId}' has 1 positional parameters, but spec has 2 declared ``` Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/fleet/common/openapi/bundled.json | 58 ------------------- .../plugins/fleet/common/openapi/bundled.yaml | 36 ------------ ...epm@packages@{pkg_name}@{pkg_version}.yaml | 11 ---- .../paths/fleet_server_hosts@{item_id}.yaml | 10 ---- .../package_policies@{package_policy_id}.yaml | 5 -- .../openapi/paths/proxies@{item_id}.yaml | 10 ---- 6 files changed, 130 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 0a62474c89056..9b994df3afe33 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -880,24 +880,6 @@ }, "operationId": "update-package", "description": "", - "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "pkgName", - "in": "path", - "required": true - }, - { - "schema": { - "type": "string" - }, - "name": "pkgVersion", - "in": "path", - "required": true - } - ], "requestBody": { "content": { "application/json": { @@ -3625,14 +3607,6 @@ } }, "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "packagePolicyId", - "in": "path", - "required": true - }, { "schema": { "type": "boolean" @@ -4273,14 +4247,6 @@ } }, "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "itemId", - "in": "path", - "required": true - }, { "$ref": "#/components/parameters/kbn_xsrf" } @@ -4333,14 +4299,6 @@ } }, "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "itemId", - "in": "path", - "required": true - }, { "$ref": "#/components/parameters/kbn_xsrf" } @@ -4503,14 +4461,6 @@ } }, "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "itemId", - "in": "path", - "required": true - }, { "$ref": "#/components/parameters/kbn_xsrf" } @@ -4569,14 +4519,6 @@ } }, "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "itemId", - "in": "path", - "required": true - }, { "$ref": "#/components/parameters/kbn_xsrf" } diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index 7b93038b146bc..65bab73457170 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -556,17 +556,6 @@ paths: - items operationId: update-package description: '' - parameters: - - schema: - type: string - name: pkgName - in: path - required: true - - schema: - type: string - name: pkgVersion - in: path - required: true requestBody: content: application/json: @@ -2247,11 +2236,6 @@ paths: required: - id parameters: - - schema: - type: string - name: packagePolicyId - in: path - required: true - schema: type: boolean name: force @@ -2652,11 +2636,6 @@ paths: required: - id parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: '#/components/parameters/kbn_xsrf' put: summary: Fleet Server Hosts - Update @@ -2688,11 +2667,6 @@ paths: required: - item parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: '#/components/parameters/kbn_xsrf' /proxies: get: @@ -2795,11 +2769,6 @@ paths: required: - id parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: '#/components/parameters/kbn_xsrf' put: summary: Fleet Proxies - Update @@ -2835,11 +2804,6 @@ paths: required: - item parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: '#/components/parameters/kbn_xsrf' /kubernetes: get: diff --git a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml index c6a5e55123591..8fe228f91bbd0 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml @@ -143,17 +143,6 @@ put: - items operationId: update-package description: '' - parameters: - - schema: - type: string - name: pkgName - in: path - required: true - - schema: - type: string - name: pkgVersion - in: path - required: true requestBody: content: application/json: diff --git a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml index b2a50f8b52b8e..141274274b840 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/fleet_server_hosts@{item_id}.yaml @@ -36,11 +36,6 @@ delete: required: - id parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: ../components/headers/kbn_xsrf.yaml put: summary: Fleet Server Hosts - Update @@ -72,9 +67,4 @@ put: required: - item parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: ../components/headers/kbn_xsrf.yaml diff --git a/x-pack/plugins/fleet/common/openapi/paths/package_policies@{package_policy_id}.yaml b/x-pack/plugins/fleet/common/openapi/paths/package_policies@{package_policy_id}.yaml index bb4a76f1cd3df..72773f43483be 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/package_policies@{package_policy_id}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/package_policies@{package_policy_id}.yaml @@ -62,11 +62,6 @@ delete: required: - id parameters: - - schema: - type: string - name: packagePolicyId - in: path - required: true - schema: type: boolean name: force diff --git a/x-pack/plugins/fleet/common/openapi/paths/proxies@{item_id}.yaml b/x-pack/plugins/fleet/common/openapi/paths/proxies@{item_id}.yaml index 3c269de04b882..882ddd5bf3f07 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/proxies@{item_id}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/proxies@{item_id}.yaml @@ -36,11 +36,6 @@ delete: required: - id parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: ../components/headers/kbn_xsrf.yaml put: summary: Fleet Proxies - Update @@ -76,9 +71,4 @@ put: required: - item parameters: - - schema: - type: string - name: itemId - in: path - required: true - $ref: ../components/headers/kbn_xsrf.yaml From 62d5a2a702268a7f807072740a9db30c620c92a2 Mon Sep 17 00:00:00 2001 From: Maxim Palenov Date: Mon, 6 Feb 2023 22:59:03 +0100 Subject: [PATCH 023/134] [Security Solution] Add clearing rules table state functionality (#150059) **Addresses:** https://github.com/elastic/kibana/issues/145968 ## Summary This PR adds functionality to clear persisted rules table state implemented in https://github.com/elastic/kibana/issues/140263. https://user-images.githubusercontent.com/3775283/216311325-e19e81e1-f232-4c16-b9df-a49fcc8c98d0.mov ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../components/rules_table/index.tsx | 2 - .../__mocks__/rules_table_context.tsx | 2 + .../rules_table/rules_table_context.test.tsx | 115 +++ .../rules_table/rules_table_context.tsx | 82 +- .../rules_table/rules_table_defaults.ts | 1 + ...initialize_rules_table_saved_state.test.ts | 626 -------------- .../use_rules_table_saved_state.test.ts | 762 ++++++++++++++++++ ...tate.ts => use_rules_table_saved_state.ts} | 99 ++- .../rules_table/rules_table_utility_bar.tsx | 10 + .../detection_engine/rules/translations.ts | 7 + 10 files changed, 1019 insertions(+), 687 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.test.tsx delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.test.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.test.ts rename x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/{use_initialize_rules_table_saved_state.ts => use_rules_table_saved_state.ts} (56%) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/index.tsx index 10d3a86b3ca5a..58ba3c2f96a5c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/index.tsx @@ -9,7 +9,6 @@ import { EuiSpacer } from '@elastic/eui'; import React from 'react'; import { useRouteSpy } from '../../../../common/utils/route/use_route_spy'; import { RulesManagementTour } from './rules_table/guided_onboarding/rules_management_tour'; -import { useInitializeRulesTableSavedState } from './rules_table/use_initialize_rules_table_saved_state'; import { useSyncRulesTableSavedState } from './rules_table/use_sync_rules_table_saved_state'; import { RulesTables } from './rules_tables'; import type { AllRulesTabs } from './rules_table_toolbar'; @@ -24,7 +23,6 @@ import { RulesTableToolbar } from './rules_table_toolbar'; * * Import/Export */ export const AllRules = React.memo(() => { - useInitializeRulesTableSavedState(); useSyncRulesTableSavedState(); const [{ tabName }] = useRouteSpy(); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/__mocks__/rules_table_context.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/__mocks__/rules_table_context.tsx index 3344b013398f3..1a03370ea5915 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/__mocks__/rules_table_context.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/__mocks__/rules_table_context.tsx @@ -40,6 +40,7 @@ export const useRulesTableContextMock = { loadingRuleIds: [], loadingRulesAction: null, selectedRuleIds: [], + isDefault: true, }, actions: { reFetchRules: jest.fn(), @@ -54,6 +55,7 @@ export const useRulesTableContextMock = { setSelectedRuleIds: jest.fn(), setSortingOptions: jest.fn(), clearRulesSelection: jest.fn(), + clearFilters: jest.fn(), }, }), }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.test.tsx new file mode 100644 index 0000000000000..552ca7b0cbd88 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.test.tsx @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import type { PropsWithChildren } from 'react'; +import React from 'react'; +import { useUiSetting$ } from '../../../../../common/lib/kibana'; +import type { RulesTableState } from './rules_table_context'; +import { RulesTableContextProvider, useRulesTableContext } from './rules_table_context'; +import { + DEFAULT_FILTER_OPTIONS, + DEFAULT_PAGE, + DEFAULT_RULES_PER_PAGE, + DEFAULT_SORTING_OPTIONS, +} from './rules_table_defaults'; +import { RuleSource } from './rules_table_saved_state'; +import { useFindRulesInMemory } from './use_find_rules_in_memory'; +import { useRulesTableSavedState } from './use_rules_table_saved_state'; + +jest.mock('../../../../../common/lib/kibana'); +jest.mock('./use_find_rules_in_memory'); +jest.mock('./use_rules_table_saved_state'); + +function renderUseRulesTableContext( + savedState: ReturnType +): RulesTableState { + (useFindRulesInMemory as jest.Mock).mockReturnValue({ + data: { rules: [], total: 0 }, + refetch: jest.fn(), + dataUpdatedAt: 0, + isFetched: false, + isFetching: false, + isLoading: false, + isRefetching: false, + }); + (useUiSetting$ as jest.Mock).mockReturnValue([{ on: false, value: 0, idleTimeout: 0 }]); + (useRulesTableSavedState as jest.Mock).mockReturnValue(savedState); + + const wrapper = ({ children }: PropsWithChildren<{}>) => ( + {children} + ); + const { + result: { + current: { state }, + }, + } = renderHook(() => useRulesTableContext(), { wrapper }); + + return state; +} + +describe('RulesTableContextProvider', () => { + describe('persisted state', () => { + it('restores persisted rules table state', () => { + const state = renderUseRulesTableContext({ + filter: { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + }, + sorting: { + field: 'name', + order: 'asc', + }, + pagination: { + page: 2, + perPage: 10, + }, + }); + + expect(state.filterOptions).toEqual({ + filter: 'test', + tags: ['test'], + showCustomRules: true, + showElasticRules: false, + enabled: true, + }); + expect(state.sortingOptions).toEqual({ + field: 'name', + order: 'asc', + }); + expect(state.pagination).toEqual({ + page: 2, + perPage: 10, + total: 0, + }); + expect(state.isDefault).toBeFalsy(); + }); + + it('restores default rules table state', () => { + const state = renderUseRulesTableContext({}); + + expect(state.filterOptions).toEqual({ + filter: DEFAULT_FILTER_OPTIONS.filter, + tags: DEFAULT_FILTER_OPTIONS.tags, + showCustomRules: DEFAULT_FILTER_OPTIONS.showCustomRules, + showElasticRules: DEFAULT_FILTER_OPTIONS.showElasticRules, + }); + expect(state.sortingOptions).toEqual({ + field: DEFAULT_SORTING_OPTIONS.field, + order: DEFAULT_SORTING_OPTIONS.order, + }); + expect(state.pagination).toEqual({ + page: DEFAULT_PAGE, + perPage: DEFAULT_RULES_PER_PAGE, + total: 0, + }); + expect(state.isDefault).toBeTruthy(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx index a66361271427a..5667b544b7b72 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx @@ -14,9 +14,12 @@ import React, { useState, useRef, } from 'react'; +import { isEqual } from 'lodash'; import { DEFAULT_RULES_TABLE_REFRESH_SETTING } from '../../../../../../common/constants'; import { invariant } from '../../../../../../common/utils/invariant'; +import { useReplaceUrlParams } from '../../../../../common/utils/global_query_string/helpers'; import { useKibana, useUiSetting$ } from '../../../../../common/lib/kibana'; +import { URL_PARAM_KEY } from '../../../../../common/hooks/use_url_state'; import type { FilterOptions, PaginationOptions, @@ -29,8 +32,11 @@ import { DEFAULT_FILTER_OPTIONS, DEFAULT_SORTING_OPTIONS, } from './rules_table_defaults'; +import { RuleSource } from './rules_table_saved_state'; import { useFindRulesInMemory } from './use_find_rules_in_memory'; +import { useRulesTableSavedState } from './use_rules_table_saved_state'; import { getRulesComparator } from './utils'; +import { RULES_TABLE_STATE_STORAGE_KEY } from '../constants'; export interface RulesTableState { /** @@ -101,6 +107,10 @@ export interface RulesTableState { * Currently selected table sorting */ sortingOptions: SortingOptions; + /** + * Whether the state has its default value + */ + isDefault: boolean; } export type LoadingRuleAction = @@ -142,6 +152,10 @@ export interface RulesTableActions { * clears rules selection on a page */ clearRulesSelection: () => void; + /** + * Clears rules table filters + */ + clearFilters: () => void; } export interface RulesTableContextType { @@ -163,13 +177,29 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide value: number; idleTimeout: number; }>(DEFAULT_RULES_TABLE_REFRESH_SETTING); - const { storage } = useKibana().services; + const { storage, sessionStorage } = useKibana().services; + const { + filter: savedFilter, + sorting: savedSorting, + pagination: savedPagination, + } = useRulesTableSavedState(); const [isInMemorySorting, setIsInMemorySorting] = useState( storage.get(IN_MEMORY_STORAGE_KEY) ?? false ); - const [filterOptions, setFilterOptions] = useState(DEFAULT_FILTER_OPTIONS); - const [sortingOptions, setSortingOptions] = useState(DEFAULT_SORTING_OPTIONS); + const [filterOptions, setFilterOptions] = useState({ + filter: savedFilter?.searchTerm ?? DEFAULT_FILTER_OPTIONS.filter, + tags: savedFilter?.tags ?? DEFAULT_FILTER_OPTIONS.tags, + showCustomRules: + savedFilter?.source === RuleSource.Custom ?? DEFAULT_FILTER_OPTIONS.showCustomRules, + showElasticRules: + savedFilter?.source === RuleSource.Prebuilt ?? DEFAULT_FILTER_OPTIONS.showElasticRules, + enabled: savedFilter?.enabled, + }); + const [sortingOptions, setSortingOptions] = useState({ + field: savedSorting?.field ?? DEFAULT_SORTING_OPTIONS.field, + order: savedSorting?.order ?? DEFAULT_SORTING_OPTIONS.order, + }); const [isAllSelected, setIsAllSelected] = useState(false); const [isRefreshOn, setIsRefreshOn] = useState(autoRefreshSettings.on); const [loadingRules, setLoadingRules] = useState({ @@ -177,8 +207,8 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide action: null, }); const [isPreflightInProgress, setIsPreflightInProgress] = useState(false); - const [page, setPage] = useState(DEFAULT_PAGE); - const [perPage, setPerPage] = useState(DEFAULT_RULES_PER_PAGE); + const [page, setPage] = useState(savedPagination?.page ?? DEFAULT_PAGE); + const [perPage, setPerPage] = useState(savedPagination?.perPage ?? DEFAULT_RULES_PER_PAGE); const [selectedRuleIds, setSelectedRuleIds] = useState([]); const autoRefreshBeforePause = useRef(null); @@ -211,6 +241,26 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide setIsAllSelected(false); }, []); + const replaceUrlParams = useReplaceUrlParams(); + const clearFilters = useCallback(() => { + setFilterOptions({ + filter: DEFAULT_FILTER_OPTIONS.filter, + showElasticRules: DEFAULT_FILTER_OPTIONS.showElasticRules, + showCustomRules: DEFAULT_FILTER_OPTIONS.showCustomRules, + tags: DEFAULT_FILTER_OPTIONS.tags, + enabled: undefined, + }); + setSortingOptions({ + field: DEFAULT_SORTING_OPTIONS.field, + order: DEFAULT_SORTING_OPTIONS.order, + }); + setPage(DEFAULT_PAGE); + setPerPage(DEFAULT_RULES_PER_PAGE); + + replaceUrlParams({ [URL_PARAM_KEY.rulesTable]: null }); + sessionStorage.remove(RULES_TABLE_STATE_STORAGE_KEY); + }, [setFilterOptions, setSortingOptions, setPage, setPerPage, replaceUrlParams, sessionStorage]); + useEffect(() => { // pause table auto refresh when any of rule selected // store current auto refresh value, to use it later, when all rules selection will be cleared @@ -262,6 +312,7 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide setSortingOptions, clearRulesSelection, setIsPreflightInProgress, + clearFilters, }), [ refetch, @@ -276,6 +327,7 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide setSortingOptions, clearRulesSelection, setIsPreflightInProgress, + clearFilters, ] ); @@ -286,7 +338,7 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide pagination: { page, perPage, - total: isInMemorySorting ? rules.length : total, + total, }, filterOptions, isPreflightInProgress, @@ -303,6 +355,11 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide loadingRulesAction: loadingRules.action, selectedRuleIds, sortingOptions, + isDefault: isDefaultState(filterOptions, sortingOptions, { + page, + perPage, + total: isInMemorySorting ? rules.length : total, + }), }, actions, }), @@ -346,3 +403,16 @@ export const useRulesTableContext = (): RulesTableContextType => { export const useRulesTableContextOptional = (): RulesTableContextType | null => useContext(RulesTableContext); + +function isDefaultState( + filter: FilterOptions, + sorting: SortingOptions, + pagination: PaginationOptions +): boolean { + return ( + isEqual(filter, DEFAULT_FILTER_OPTIONS) && + isEqual(sorting, DEFAULT_SORTING_OPTIONS) && + pagination.page === DEFAULT_PAGE && + pagination.perPage === DEFAULT_RULES_PER_PAGE + ); +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_defaults.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_defaults.ts index 4cee292159691..6694b0a3a4ecd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_defaults.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_defaults.ts @@ -12,6 +12,7 @@ export const DEFAULT_FILTER_OPTIONS: FilterOptions = { tags: [], showCustomRules: false, showElasticRules: false, + enabled: undefined, }; export const DEFAULT_SORTING_OPTIONS: SortingOptions = { field: 'enabled', diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.test.ts deleted file mode 100644 index b7a594034f034..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.test.ts +++ /dev/null @@ -1,626 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { renderHook } from '@testing-library/react-hooks'; -import { RULES_TABLE_MAX_PAGE_SIZE } from '../../../../../../common/constants'; -import { useRulesTableContextMock } from './__mocks__/rules_table_context'; -import { useInitializeRulesTableSavedState } from './use_initialize_rules_table_saved_state'; -import type { - RulesTableStorageSavedState, - RulesTableUrlSavedState, -} from './rules_table_saved_state'; -import { RuleSource } from './rules_table_saved_state'; -import { DEFAULT_FILTER_OPTIONS, DEFAULT_SORTING_OPTIONS } from './rules_table_defaults'; -import { useRulesTableContext } from './rules_table_context'; -import { mockRulesTablePersistedState } from './__mocks__/mock_rules_table_persistent_state'; - -jest.mock('../../../../../common/lib/kibana'); -jest.mock('../../../../../common/utils/global_query_string/helpers'); -jest.mock('./rules_table_context'); - -describe('useInitializeRulesTableSavedState', () => { - const urlSavedState: RulesTableUrlSavedState = { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - order: 'asc', - page: 2, - perPage: 10, - enabled: true, - }; - const storageSavedState: RulesTableStorageSavedState = { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - order: 'asc', - perPage: 20, - enabled: false, - }; - const rulesTableContext = useRulesTableContextMock.create(); - const actions = rulesTableContext.actions; - - beforeEach(() => { - jest.clearAllMocks(); - (useRulesTableContext as jest.Mock).mockReturnValue(rulesTableContext); - }); - - describe('when state is not saved', () => { - beforeEach(() => { - mockRulesTablePersistedState({ urlState: null, storageState: null }); - }); - - it('does not restore the state', () => { - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).not.toHaveBeenCalled(); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - }); - - describe('when state is saved in the url', () => { - beforeEach(() => { - mockRulesTablePersistedState({ urlState: urlSavedState, storageState: null }); - }); - - it('restores the state', () => { - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: urlSavedState.searchTerm, - showCustomRules: urlSavedState.source === RuleSource.Custom, - showElasticRules: urlSavedState.source === RuleSource.Prebuilt, - tags: urlSavedState.tags, - enabled: urlSavedState.enabled, - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: urlSavedState.field, - order: urlSavedState.order, - }); - expect(actions.setPage).toHaveBeenCalledWith(urlSavedState.page); - expect(actions.setPerPage).toHaveBeenCalledWith(urlSavedState.perPage); - }); - - it('restores the state ignoring negative page size', () => { - mockRulesTablePersistedState({ - urlState: { ...urlSavedState, perPage: -1 }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores the state ignoring the page size larger than max allowed', () => { - mockRulesTablePersistedState({ - urlState: { ...urlSavedState, perPage: RULES_TABLE_MAX_PAGE_SIZE + 1 }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - }); - - describe('when partial state is saved in the url', () => { - it('restores only the search term', () => { - mockRulesTablePersistedState({ urlState: { searchTerm: 'test' }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - filter: 'test', - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only show prebuilt rules filter', () => { - mockRulesTablePersistedState({ - urlState: { source: RuleSource.Prebuilt }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - showElasticRules: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only show custom rules filter', () => { - mockRulesTablePersistedState({ urlState: { source: RuleSource.Custom }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - showCustomRules: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only tags', () => { - mockRulesTablePersistedState({ urlState: { tags: ['test'] }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - tags: ['test'], - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only enabled state filter', () => { - mockRulesTablePersistedState({ urlState: { enabled: true }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - enabled: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only sorting field', () => { - mockRulesTablePersistedState({ urlState: { field: 'name' }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: 'name', - order: DEFAULT_SORTING_OPTIONS.order, - }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only sorting order', () => { - mockRulesTablePersistedState({ urlState: { order: 'asc' }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: DEFAULT_SORTING_OPTIONS.field, - order: 'asc', - }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only page number', () => { - mockRulesTablePersistedState({ urlState: { page: 10 }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).toHaveBeenCalledWith(10); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only page size', () => { - mockRulesTablePersistedState({ urlState: { perPage: 10 }, storageState: null }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - }); - - describe('when state is saved in the storage', () => { - beforeEach(() => { - mockRulesTablePersistedState({ urlState: null, storageState: storageSavedState }); - }); - - it('restores the state', () => { - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: storageSavedState.searchTerm, - showCustomRules: storageSavedState.source === RuleSource.Custom, - showElasticRules: storageSavedState.source === RuleSource.Prebuilt, - tags: storageSavedState.tags, - enabled: storageSavedState.enabled, - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: storageSavedState.field, - order: storageSavedState.order, - }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).toHaveBeenCalledWith(storageSavedState.perPage); - }); - - it('restores the state ignoring negative page size', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { ...storageSavedState, perPage: -1 }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores the state ignoring the page size larger than max allowed', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { ...storageSavedState, perPage: RULES_TABLE_MAX_PAGE_SIZE + 1 }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - }); - - describe('when partial state is saved in the storage', () => { - it('restores only the search term', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { searchTerm: 'test' } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - filter: 'test', - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only show prebuilt rules filter', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { source: RuleSource.Prebuilt }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - showElasticRules: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only show custom rules filter', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { source: RuleSource.Custom } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - showCustomRules: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only tags', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { tags: ['test'] } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - tags: ['test'], - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only enabled state filter', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { enabled: true } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - enabled: true, - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only sorting field', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { field: 'name' } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: 'name', - order: DEFAULT_SORTING_OPTIONS.order, - }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only sorting order', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { order: 'asc' } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: DEFAULT_SORTING_OPTIONS.field, - order: 'asc', - }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('does not restore the page number', () => { - mockRulesTablePersistedState({ - urlState: null, - // @ts-expect-error Passing an invalid value for the test - storageState: { page: 10 }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - - it('restores only page size', () => { - mockRulesTablePersistedState({ urlState: null, storageState: { perPage: 10 } }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith(DEFAULT_FILTER_OPTIONS); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - }); - - describe('when state is saved in the url and the storage', () => { - beforeEach(() => { - mockRulesTablePersistedState({ urlState: urlSavedState, storageState: storageSavedState }); - }); - - it('restores the state from the url', () => { - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: urlSavedState.searchTerm, - showCustomRules: urlSavedState.source === RuleSource.Custom, - showElasticRules: urlSavedState.source === RuleSource.Prebuilt, - tags: urlSavedState.tags, - enabled: urlSavedState.enabled, - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - field: urlSavedState.field, - order: urlSavedState.order, - }); - expect(actions.setPage).toHaveBeenCalledWith(urlSavedState.page); - expect(actions.setPerPage).toHaveBeenCalledWith(urlSavedState.perPage); - }); - }); - - describe('when partial state is saved in the url and in the storage', () => { - it('restores only the search term', () => { - mockRulesTablePersistedState({ - urlState: { searchTerm: 'test' }, - storageState: { field: 'name' }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - ...DEFAULT_FILTER_OPTIONS, - filter: 'test', - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ - ...DEFAULT_SORTING_OPTIONS, - field: 'name', - }); - }); - }); - - describe('when there is invalid state in the url', () => { - it('does not restore the filter', () => { - mockRulesTablePersistedState({ - urlState: { - searchTerm: 'test', - source: RuleSource.Custom, - // @ts-expect-error Passing an invalid value for the test - tags: [1, 2, 3], - field: 'name', - order: 'asc', - page: 2, - perPage: 10, - }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: '', - showCustomRules: false, - showElasticRules: false, - tags: [], - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ field: 'name', order: 'asc' }); - expect(actions.setPage).toHaveBeenCalledWith(2); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - - it('does not restore the sorting', () => { - mockRulesTablePersistedState({ - urlState: { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - // @ts-expect-error Passing an invalid value for the test - order: 'abc', - page: 2, - perPage: 10, - }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: 'test', - showCustomRules: true, - showElasticRules: false, - tags: ['test'], - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).toHaveBeenCalledWith(2); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - - it('does not restore the pagination', () => { - mockRulesTablePersistedState({ - urlState: { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - order: 'asc', - // @ts-expect-error Passing an invalid value for the test - page: 'aaa', - perPage: 10, - }, - storageState: null, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: 'test', - showCustomRules: true, - showElasticRules: false, - tags: ['test'], - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ field: 'name', order: 'asc' }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - }); - - describe('when there is invalid state in the storage', () => { - it('does not restore the filter', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { - searchTerm: 'test', - source: RuleSource.Custom, - // @ts-expect-error Passing an invalid value for the test - tags: [1, 2, 3], - field: 'name', - order: 'asc', - perPage: 10, - }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: '', - showCustomRules: false, - showElasticRules: false, - tags: [], - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ field: 'name', order: 'asc' }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - - it('does not restore the sorting', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - // @ts-expect-error Passing an invalid value for the test - order: 'abc', - perPage: 10, - }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: 'test', - showCustomRules: true, - showElasticRules: false, - tags: ['test'], - }); - expect(actions.setSortingOptions).not.toHaveBeenCalled(); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).toHaveBeenCalledWith(10); - }); - - it('does not restore the pagination', () => { - mockRulesTablePersistedState({ - urlState: null, - storageState: { - searchTerm: 'test', - source: RuleSource.Custom, - tags: ['test'], - field: 'name', - order: 'asc', - // @ts-expect-error Passing an invalid value for the test - perPage: 'aaa', - }, - }); - - renderHook(() => useInitializeRulesTableSavedState()); - - expect(actions.setFilterOptions).toHaveBeenCalledWith({ - filter: 'test', - showCustomRules: true, - showElasticRules: false, - tags: ['test'], - }); - expect(actions.setSortingOptions).toHaveBeenCalledWith({ field: 'name', order: 'asc' }); - expect(actions.setPage).not.toHaveBeenCalled(); - expect(actions.setPerPage).not.toHaveBeenCalled(); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.test.ts new file mode 100644 index 0000000000000..de9bf77e13236 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.test.ts @@ -0,0 +1,762 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { RULES_TABLE_MAX_PAGE_SIZE } from '../../../../../../common/constants'; +import type { + RulesTableStorageSavedState, + RulesTableUrlSavedState, +} from './rules_table_saved_state'; +import { RuleSource } from './rules_table_saved_state'; +import { mockRulesTablePersistedState } from './__mocks__/mock_rules_table_persistent_state'; +import { useRulesTableSavedState } from './use_rules_table_saved_state'; + +jest.mock('../../../../../common/lib/kibana'); +jest.mock('../../../../../common/utils/global_query_string/helpers'); +jest.mock('./rules_table_context'); + +describe('useRulesTableSavedState', () => { + const urlSavedState: RulesTableUrlSavedState = { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + field: 'name', + order: 'asc', + page: 2, + perPage: 10, + }; + const storageSavedState: RulesTableStorageSavedState = { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: false, + field: 'name', + order: 'asc', + perPage: 20, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when the state is not saved', () => { + beforeEach(() => { + mockRulesTablePersistedState({ urlState: null, storageState: null }); + }); + + it('does not return the state', () => { + const { + result: { current: currentResult }, + } = renderHook(() => useRulesTableSavedState()); + + expect(currentResult).toEqual({}); + }); + }); + + describe('when the state is saved in the url', () => { + beforeEach(() => { + mockRulesTablePersistedState({ urlState: urlSavedState, storageState: null }); + }); + + it('returns the state', () => { + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: urlSavedState.searchTerm, + source: urlSavedState.source, + tags: urlSavedState.tags, + enabled: urlSavedState.enabled, + }); + expect(sorting).toEqual({ + field: urlSavedState.field, + order: urlSavedState.order, + }); + expect(pagination).toEqual({ + page: urlSavedState.page, + perPage: urlSavedState.perPage, + }); + }); + + it('returns the state ignoring negative page size', () => { + mockRulesTablePersistedState({ + urlState: { ...urlSavedState, perPage: -1 }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: urlSavedState.searchTerm, + source: urlSavedState.source, + tags: urlSavedState.tags, + enabled: urlSavedState.enabled, + }); + expect(sorting).toEqual({ + field: urlSavedState.field, + order: urlSavedState.order, + }); + expect(pagination).toEqual({}); + }); + + it('returns the state ignoring the page size larger than max allowed', () => { + mockRulesTablePersistedState({ + urlState: { ...urlSavedState, perPage: RULES_TABLE_MAX_PAGE_SIZE + 1 }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: urlSavedState.searchTerm, + source: urlSavedState.source, + tags: urlSavedState.tags, + enabled: urlSavedState.enabled, + }); + expect(sorting).toEqual({ + field: urlSavedState.field, + order: urlSavedState.order, + }); + expect(pagination).toEqual({ + page: urlSavedState.page, + }); + }); + }); + + describe('when the partial state is saved in the url', () => { + it('returns only the search term', () => { + mockRulesTablePersistedState({ urlState: { searchTerm: 'test' }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only show prebuilt rules filter', () => { + mockRulesTablePersistedState({ + urlState: { source: RuleSource.Prebuilt }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + source: RuleSource.Prebuilt, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only show custom rules filter', () => { + mockRulesTablePersistedState({ urlState: { source: RuleSource.Custom }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + source: RuleSource.Custom, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only tags', () => { + mockRulesTablePersistedState({ urlState: { tags: ['test'] }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + tags: ['test'], + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only enabled state', () => { + mockRulesTablePersistedState({ urlState: { enabled: true }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + enabled: true, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only sorting field', () => { + mockRulesTablePersistedState({ urlState: { field: 'name' }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ field: 'name' }); + expect(pagination).toEqual({}); + }); + + it('returns only sorting order', () => { + mockRulesTablePersistedState({ urlState: { order: 'asc' }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ order: 'asc' }); + expect(pagination).toEqual({}); + }); + + it('returns only page number', () => { + mockRulesTablePersistedState({ urlState: { page: 10 }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({}); + expect(pagination).toEqual({ page: 10 }); + }); + + it('returns only page size', () => { + mockRulesTablePersistedState({ urlState: { perPage: 10 }, storageState: null }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({}); + expect(pagination).toEqual({ perPage: 10 }); + }); + }); + + describe('when state is saved in the storage', () => { + beforeEach(() => { + mockRulesTablePersistedState({ urlState: null, storageState: storageSavedState }); + }); + + it('returns the state', () => { + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: storageSavedState.searchTerm, + source: storageSavedState.source, + tags: storageSavedState.tags, + enabled: storageSavedState.enabled, + }); + expect(sorting).toEqual({ + field: storageSavedState.field, + order: storageSavedState.order, + }); + expect(pagination).toEqual({ + perPage: storageSavedState.perPage, + }); + }); + + it('returns the state ignoring negative page size', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { ...storageSavedState, perPage: -1 }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: storageSavedState.searchTerm, + source: storageSavedState.source, + tags: storageSavedState.tags, + enabled: storageSavedState.enabled, + }); + expect(sorting).toEqual({ + field: storageSavedState.field, + order: storageSavedState.order, + }); + expect(pagination).toEqual({}); + }); + + it('returns the state ignoring the page size larger than max allowed', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { ...storageSavedState, perPage: RULES_TABLE_MAX_PAGE_SIZE + 1 }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: storageSavedState.searchTerm, + source: storageSavedState.source, + tags: storageSavedState.tags, + enabled: storageSavedState.enabled, + }); + expect(sorting).toEqual({ + field: storageSavedState.field, + order: storageSavedState.order, + }); + expect(pagination).toEqual({}); + }); + }); + + describe('when partial state is saved in the storage', () => { + it('returns only the search term', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { searchTerm: 'test' } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only show prebuilt rules filter', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { source: RuleSource.Prebuilt }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + source: RuleSource.Prebuilt, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only show custom rules filter', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { source: RuleSource.Custom } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + source: RuleSource.Custom, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only tags', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { tags: ['test'] } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + tags: ['test'], + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only enabled state', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { enabled: true } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + enabled: true, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only sorting field', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { field: 'name' } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ field: 'name' }); + expect(pagination).toEqual({}); + }); + + it('returns only sorting order', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { order: 'asc' } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ order: 'asc' }); + expect(pagination).toEqual({}); + }); + + it('does not return the page number', () => { + mockRulesTablePersistedState({ + urlState: null, + // @ts-expect-error Passing an invalid value for the test + storageState: { page: 10 }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({}); + expect(pagination).toEqual({}); + }); + + it('returns only page size', () => { + mockRulesTablePersistedState({ urlState: null, storageState: { perPage: 10 } }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({}); + expect(pagination).toEqual({ perPage: 10 }); + }); + }); + + describe('when state is saved in the url and the storage', () => { + beforeEach(() => { + mockRulesTablePersistedState({ urlState: urlSavedState, storageState: storageSavedState }); + }); + + it('returns the state from the url', () => { + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: urlSavedState.searchTerm, + source: urlSavedState.source, + tags: urlSavedState.tags, + enabled: urlSavedState.enabled, + }); + expect(sorting).toEqual({ + field: urlSavedState.field, + order: urlSavedState.order, + }); + expect(pagination).toEqual({ + page: urlSavedState.page, + perPage: urlSavedState.perPage, + }); + }); + }); + + describe('when partial state is saved in the url and in the storage', () => { + it('returns only the search term', () => { + mockRulesTablePersistedState({ + urlState: { searchTerm: 'test' }, + storageState: { field: 'name' }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + }); + expect(sorting).toEqual({ + field: 'name', + }); + expect(pagination).toEqual({}); + }); + }); + + describe('when there is invalid state in the url', () => { + it('does not return the filter', () => { + mockRulesTablePersistedState({ + urlState: { + searchTerm: 'test', + source: RuleSource.Custom, + // @ts-expect-error Passing an invalid value for the test + tags: [1, 2, 3], + enabled: true, + field: 'name', + order: 'asc', + page: 2, + perPage: 10, + }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ + field: 'name', + order: 'asc', + }); + expect(pagination).toEqual({ + page: 2, + perPage: 10, + }); + }); + + it('does not return the sorting', () => { + mockRulesTablePersistedState({ + urlState: { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + field: 'name', + // @ts-expect-error Passing an invalid value for the test + order: 'abc', + page: 2, + perPage: 10, + }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({ + page: 2, + perPage: 10, + }); + }); + + it('does not return the pagination', () => { + mockRulesTablePersistedState({ + urlState: { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + field: 'name', + order: 'asc', + // @ts-expect-error Passing an invalid value for the test + page: 'aaa', + perPage: 10, + }, + storageState: null, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + }); + expect(sorting).toEqual({ + field: 'name', + order: 'asc', + }); + expect(pagination).toEqual({}); + }); + }); + + describe('when there is invalid state in the storage', () => { + it('does not return the filter', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { + searchTerm: 'test', + source: RuleSource.Custom, + // @ts-expect-error Passing an invalid value for the test + tags: [1, 2, 3], + enabled: true, + field: 'name', + order: 'asc', + perPage: 10, + }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({}); + expect(sorting).toEqual({ + field: 'name', + order: 'asc', + }); + expect(pagination).toEqual({ + perPage: 10, + }); + }); + + it('does not return the sorting', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + field: 'name', + // @ts-expect-error Passing an invalid value for the test + order: 'abc', + perPage: 10, + }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + }); + expect(sorting).toEqual({}); + expect(pagination).toEqual({ + perPage: 10, + }); + }); + + it('does not return the pagination', () => { + mockRulesTablePersistedState({ + urlState: null, + storageState: { + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + field: 'name', + order: 'asc', + // @ts-expect-error Passing an invalid value for the test + perPage: 'aaa', + }, + }); + + const { + result: { + current: { filter, sorting, pagination }, + }, + } = renderHook(() => useRulesTableSavedState()); + + expect(filter).toEqual({ + searchTerm: 'test', + source: RuleSource.Custom, + tags: ['test'], + enabled: true, + }); + expect(sorting).toEqual({ + field: 'name', + order: 'asc', + }); + expect(pagination).toEqual({}); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.ts similarity index 56% rename from x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.ts index d70547dc74548..5cc9fe604ce7f 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_initialize_rules_table_saved_state.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/use_rules_table_saved_state.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { useEffect } from 'react'; import type { Storage } from '@kbn/kibana-utils-plugin/public'; import { validateNonExact } from '@kbn/securitysolution-io-ts-utils'; import { useGetInitialUrlParamValue } from '../../../../../common/utils/global_query_string/helpers'; @@ -13,19 +12,16 @@ import { RULES_TABLE_MAX_PAGE_SIZE } from '../../../../../../common/constants'; import { useKibana } from '../../../../../common/lib/kibana'; import { URL_PARAM_KEY } from '../../../../../common/hooks/use_url_state'; import { RULES_TABLE_STATE_STORAGE_KEY } from '../constants'; -import { useRulesTableContext } from './rules_table_context'; import type { RulesTableStorageSavedState, RulesTableUrlSavedState, } from './rules_table_saved_state'; import { - RuleSource, RulesTableSavedFilter, RulesTableStorageSavedPagination, RulesTableUrlSavedPagination, RulesTableSavedSorting, } from './rules_table_saved_state'; -import { DEFAULT_FILTER_OPTIONS, DEFAULT_SORTING_OPTIONS } from './rules_table_defaults'; function readStorageState(storage: Storage): RulesTableStorageSavedState | null { try { @@ -35,67 +31,64 @@ function readStorageState(storage: Storage): RulesTableStorageSavedState | null } } +export function useRulesTableSavedState(): { + filter?: RulesTableSavedFilter; + sorting?: RulesTableSavedSorting; + pagination?: RulesTableUrlSavedPagination; +} { + const getUrlParam = useGetInitialUrlParamValue(URL_PARAM_KEY.rulesTable); + const { + services: { sessionStorage }, + } = useKibana(); + + const urlState = getUrlParam(); + const storageState = readStorageState(sessionStorage); + + if (!urlState && !storageState) { + return {}; + } + + const [filter, sorting, pagination] = validateState(urlState, storageState); + + return { filter, sorting, pagination }; +} + function validateState( urlState: RulesTableUrlSavedState | null, storageState: RulesTableStorageSavedState | null ): [RulesTableSavedFilter, RulesTableSavedSorting, RulesTableUrlSavedPagination] { const [filterFromUrl] = validateNonExact(urlState, RulesTableSavedFilter); const [filterFromStorage] = validateNonExact(storageState, RulesTableSavedFilter); - const filter = { ...filterFromStorage, ...filterFromUrl }; + // We have to expose filter, sorting and pagination objects by explicitly specifying each field + // since urlState and/or storageState may contain unnecessary fields (e.g. outdated or explicitly added by user) + // and validateNonExact doesn't truncate fields not included in the type RulesTableSavedFilter and etc. + const filter: RulesTableSavedFilter = { + searchTerm: filterFromUrl?.searchTerm ?? filterFromStorage?.searchTerm, + source: filterFromUrl?.source ?? filterFromStorage?.source, + tags: filterFromUrl?.tags ?? filterFromStorage?.tags, + enabled: filterFromUrl?.enabled ?? filterFromStorage?.enabled, + }; const [sortingFromUrl] = validateNonExact(urlState, RulesTableSavedSorting); const [sortingFromStorage] = validateNonExact(storageState, RulesTableSavedSorting); - const sorting = { ...sortingFromStorage, ...sortingFromUrl }; + const sorting = { + field: sortingFromUrl?.field ?? sortingFromStorage?.field, + order: sortingFromUrl?.order ?? sortingFromStorage?.order, + }; const [paginationFromUrl] = validateNonExact(urlState, RulesTableUrlSavedPagination); const [paginationFromStorage] = validateNonExact(storageState, RulesTableStorageSavedPagination); - const pagination = { perPage: paginationFromStorage?.perPage, ...paginationFromUrl }; - - return [filter, sorting, pagination]; -} - -export function useInitializeRulesTableSavedState(): void { - const getUrlParam = useGetInitialUrlParamValue(URL_PARAM_KEY.rulesTable); - const { actions } = useRulesTableContext(); - const { - services: { sessionStorage }, - } = useKibana(); - - useEffect(() => { - const urlState = getUrlParam(); - const storageState = readStorageState(sessionStorage); + const pagination = { + page: paginationFromUrl?.page, // We don't persist page number in the session storage since it may be outdated when restored + perPage: paginationFromUrl?.perPage ?? paginationFromStorage?.perPage, + }; - if (!urlState && !storageState) { - return; - } - - const [filter, sorting, pagination] = validateState(urlState, storageState); - - actions.setFilterOptions({ - filter: filter.searchTerm ?? DEFAULT_FILTER_OPTIONS.filter, - showElasticRules: filter.source === RuleSource.Prebuilt, - showCustomRules: filter.source === RuleSource.Custom, - tags: Array.isArray(filter.tags) ? filter.tags : DEFAULT_FILTER_OPTIONS.tags, - enabled: filter.enabled, - }); - - if (sorting.field || sorting.order) { - actions.setSortingOptions({ - field: sorting.field ?? DEFAULT_SORTING_OPTIONS.field, - order: sorting.order ?? DEFAULT_SORTING_OPTIONS.order, - }); - } - - if (pagination.page) { - actions.setPage(pagination.page); - } + if ( + pagination.perPage && + (pagination.perPage < 0 || pagination.perPage > RULES_TABLE_MAX_PAGE_SIZE) + ) { + delete pagination.perPage; + } - if ( - pagination.perPage && - pagination.perPage > 0 && - pagination.perPage <= RULES_TABLE_MAX_PAGE_SIZE - ) { - actions.setPerPage(pagination.perPage); - } - }, [getUrlParam, actions, sessionStorage]); + return [filter, sorting, pagination]; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table_utility_bar.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table_utility_bar.tsx index 7c91e27423697..9c681d95ae520 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table_utility_bar.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table_utility_bar.tsx @@ -179,6 +179,16 @@ export const RulesTableUtilityBar = React.memo( > {i18n.REFRESH_RULE_POPOVER_LABEL} + {!rulesTableContext.state.isDefault && ( + + {i18n.CLEAR_RULES_TABLE_FILTERS} + + )} diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts index 0e1c1b1c0b43e..c71cc4cf267a4 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts @@ -889,6 +889,13 @@ export const REFRESH_RULE_POPOVER_LABEL = i18n.translate( } ); +export const CLEAR_RULES_TABLE_FILTERS = i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.clearRulesTableFilters', + { + defaultMessage: 'Clear filters', + } +); + /** * Bulk Export */ From 78b16cbe12cbb3f34004b8260502dea155076dbc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:41:29 -0600 Subject: [PATCH 024/134] Update dependency @babel/generator to ^7.20.14 (main) (#150262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [@babel/generator](https://babel.dev/docs/en/next/babel-generator) ([source](https://togithub.com/babel/babel)) | [`^7.20.7` -> `^7.20.14`](https://renovatebot.com/diffs/npm/@babel%2fgenerator/7.20.7/7.20.14) | [![age](https://badges.renovateapi.com/packages/npm/@babel%2fgenerator/7.20.14/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/npm/@babel%2fgenerator/7.20.14/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/npm/@babel%2fgenerator/7.20.14/compatibility-slim/7.20.7)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/npm/@babel%2fgenerator/7.20.14/confidence-slim/7.20.7)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
babel/babel ### [`v7.20.14`](https://togithub.com/babel/babel/blob/HEAD/CHANGELOG.md#v72014-2023-01-27) [Compare Source](https://togithub.com/babel/babel/compare/v7.20.7...v7.20.14) ##### :bug: Bug Fix - `babel-plugin-transform-block-scoping` - [#​15361](https://togithub.com/babel/babel/pull/15361) fix: Identifiers in the loop are not renamed ([@​liuxingbaoyu](https://togithub.com/liuxingbaoyu)) - `babel-cli`, `babel-core`, `babel-generator`, `babel-helper-transform-fixture-test-runner`, `babel-plugin-transform-destructuring`, `babel-plugin-transform-modules-commonjs`, `babel-plugin-transform-react-jsx`, `babel-traverse` - [#​15365](https://togithub.com/babel/babel/pull/15365) fix: Properly generate source maps for manually added multi-line content ([@​liuxingbaoyu](https://togithub.com/liuxingbaoyu))
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://app.renovatebot.com/dashboard#github/elastic/kibana). --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Jonathan Budzenski --- package.json | 2 +- yarn.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index c2613ebda6e76..d26b6bd79b0c2 100644 --- a/package.json +++ b/package.json @@ -669,7 +669,7 @@ "@babel/core": "^7.20.12", "@babel/eslint-parser": "^7.19.1", "@babel/eslint-plugin": "^7.19.1", - "@babel/generator": "^7.20.7", + "@babel/generator": "^7.20.14", "@babel/helper-plugin-utils": "^7.20.2", "@babel/parser": "^7.20.13", "@babel/plugin-proposal-class-properties": "^7.18.6", diff --git a/yarn.lock b/yarn.lock index 77b529bdd0bcb..fa46dea2940e9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -150,10 +150,10 @@ dependencies: eslint-rule-composer "^0.3.0" -"@babel/generator@^7.12.11", "@babel/generator@^7.12.5", "@babel/generator@^7.20.7", "@babel/generator@^7.7.2": - version "7.20.7" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.7.tgz#f8ef57c8242665c5929fe2e8d82ba75460187b4a" - integrity sha512-7wqMOJq8doJMZmP4ApXTzLxSr7+oO2jroJURrVEp6XShrQUObV8Tq/D0NCcoYg2uHqUrjzO0zwBjoYzelxK+sw== +"@babel/generator@^7.12.11", "@babel/generator@^7.12.5", "@babel/generator@^7.20.14", "@babel/generator@^7.20.7", "@babel/generator@^7.7.2": + version "7.20.14" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.20.14.tgz#9fa772c9f86a46c6ac9b321039400712b96f64ce" + integrity sha512-AEmuXHdcD3A52HHXxaTmYlb8q/xMEhoRP67B3T4Oq7lbmSoqroMZzjnGj3+i1io3pdnF8iBYVu4Ilj+c4hBxYg== dependencies: "@babel/types" "^7.20.7" "@jridgewell/gen-mapping" "^0.3.2" @@ -28475,12 +28475,12 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" -ws@8.9.0, ws@^8.2.3, ws@^8.4.2, ws@^8.9.0: +ws@8.9.0: version "8.9.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.9.0.tgz#2a994bb67144be1b53fe2d23c53c028adeb7f45e" integrity sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg== -ws@>=8.11.0: +ws@>=8.11.0, ws@^8.2.3, ws@^8.4.2, ws@^8.9.0: version "8.12.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.12.0.tgz#485074cc392689da78e1828a9ff23585e06cddd8" integrity sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig== From 55b66e20fecb39ff0026c53c3ae6b3f8242be734 Mon Sep 17 00:00:00 2001 From: Hannah Mudge Date: Mon, 6 Feb 2023 16:13:28 -0700 Subject: [PATCH 025/134] [Dashboard] [Controls] Load more options list suggestions on scroll (#148331) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/elastic/kibana/issues/140175 Closes https://github.com/elastic/kibana/issues/143580 ## Summary Oh, boy! Get ready for a doozy of a PR, folks! Let's talk about the three major things that were accomplished here: ### 1) Pagination Originally, this PR was meant to add traditional pagination to the options list control. However, after implementing a version of this, it became apparent that, not only was UI becoming uncomfortably messy, it also had some UX concerns because we were deviating from the usual pagination pattern by showing the cardinality rather than the number of pages:

So, instead of traditional pagination, we decided to take a different approach (which was made possible by https://github.com/elastic/kibana/pull/148420) - **load more options when the user scrolls to the bottom!** Here it is in action:

It is important that the first query remains **fast** - that is why we still only request the top 10 options when the control first loads. So, having a "load more" is the best approach that allows users to see more suggestions while also ensuring that the performance of options lists (especially with respect to chaining) is not impacted. Note that it is **not possible** to grab every single value of a field - the limit is `10,000`. However, since it is impractical that a user would want to scroll through `10,000` suggestions (and potentially very slow to fetch), we have instead made the limit of this "show more" functionality `1,000`. To make this clear, if the field has more than `1,000` values and the user scrolls all the way to the bottom, they will get the following message:

### 2) Cardinality Previously, the cardinality of the options list control was **only** shown as part of the control placeholder text - this meant that, once the user entered their search term, they could no longer see the cardinality of the returned options. This PR changes this functionality by placing the cardinality in a badge **beside** the search bar - this value now changes as the user types, so they can very clearly see how many options match their search:

> **Note** > After some initial feedback, we have removed both the cardinality and invalid selections badges in favour of displaying the cardinality below the search bar, like so: > >

> > So, please be aware that the screenshots above are outdated. ### 3) Changes to Queries This is where things get.... messy! Essentially, our previous queries were all built with the expectation that the Elasticsearch setting `search.allow_expensive_queries` was **off** - this meant that they worked regardless of the value of this setting. However, when trying to get the cardinality to update based on a search term, it became apparent that this was not possible if we kept the same assumptions - specifically, if `search.allow_expensive_queries` is off, there is absolutely no way for the cardinality of **keyword only fields** to respond to a search term. After a whole lot of discussion, we decided that the updating cardinality was a feature important enough to justify having **two separate versions** of the queries: 1. **Queries for when `search.allow_expensive_queries` is off**: These are essentially the same as our old queries - however, since we can safely assume that this setting is **usually** on (it defaults on, and there is no UI to easily change it), we opted to simplify them a bit. First of all, we used to create a special object for tracking the parent/child relationship of fields that are mapped as keyword+text - this was so that, if a user created a control on these fields, we could support case-insensitive search. We no longer do this - if `search.allow_expensive_queries` is off and you create a control on a text+keyword field, the search will be case sensitive. This helps clean up our code quite a bit. Second, we are no longer returning **any** cardinality. Since the cardinality is now displayed as a badge beside the search bar, users would expect that this value would change as they type - however, since it's impossible to make this happen for keyword-only fields and to keep behaviour consistent, we have opted to simply remove this badge when `search.allow_expensive_queries` is off **regardless** of the field type. So, there is no longer a need to include the `cardinality` query when grabbing the suggestions. Finally, we do not support "load more" when `search.allow_expensive_queries` is off. While this would theoretically be possible, because we are no longer grabbing the cardinality, we would have to always fetch `1,000` results when the user loads more, even if the true cardinality is much smaller. Again, we are pretty confident that **more often than not**, the `search.allow_expensive_queries` is on; therefore, we are choosing to favour developer experience in this instance because the impact should be quite small. 2. **Queries for when `search.allow_expensive_queries` is on**: When this setting is on, we now have access to the prefix query, which greatly simplifies how our queries are handled - now, rather than having separate queries for keyword-only, keyword+text, and nested fields, these have all been combined into a single query! And even better - :star: now **all** string-based fields support case-insensitive search! :star: Yup, that's right - even keyword-only fields 💃 There has been [discussion on the Elasticsearch side ](https://github.com/elastic/elasticsearch/issues/90898) about whether or not this setting is even **practical**, and so it is possible that, in the near future, this distinction will no longer be necessary. With this in mind, I have made these two versions of our queries **completely separate** from each other - while this introduces some code duplication, it makes the cleanup that may follow much, much easier. Well, that was sure fun, hey?

## How to Test I've created a quick little Python program to ingest some good testing data for this PR: ```python import random import time import pandas as pd from faker import Faker from elasticsearch import Elasticsearch SIZE = 10000 ELASTIC_PASSWORD = "changeme" INDEX_NAME = 'test_large_index' Faker.seed(time.time()) faker = Faker() hundredRandomSentences = [faker.sentence(random.randint(5, 35)) for _ in range(100)] thousandRandomIps = [faker.ipv4() if random.randint(0, 99) < 50 else faker.ipv6() for _ in range(1000)] client = Elasticsearch( "http://localhost:9200", basic_auth=("elastic", ELASTIC_PASSWORD), ) if(client.indices.exists(index=INDEX_NAME)): client.indices.delete(index=INDEX_NAME) client.indices.create(index=INDEX_NAME, mappings={"properties":{"keyword_field":{"type":"keyword"},"id":{"type":"long"},"ip_field":{"type":"ip"},"boolean_field":{"type":"boolean"},"keyword_text_field":{"type":"text","fields":{"keyword":{"type":"keyword"}}},"nested_field":{"type":"nested","properties":{"first":{"type":"text","fields":{"keyword":{"type":"keyword"}}},"last":{"type":"text","fields":{"keyword":{"type":"keyword"}}}}},"long_keyword_text_field":{"type":"text","fields":{"keyword":{"type":"keyword"}}}}}) print('Generating data', end='') for i in range(SIZE): name1 = faker.name(); [first_name1, last_name1] = name1.split(' ', 1) name2 = faker.name(); [first_name2, last_name2] = name2.split(' ', 1) response = client.create(index=INDEX_NAME, id=i, document={ 'keyword_field': faker.country(), 'id': i, 'boolean_field': faker.boolean(), 'ip_field': thousandRandomIps[random.randint(0, 999)], 'keyword_text_field': faker.name(), 'nested_field': [ { 'first': first_name1, 'last': last_name1}, { 'first': first_name2, 'last': last_name2} ], 'long_keyword_text_field': hundredRandomSentences[random.randint(0, 99)] }) print('.', end='') print(' Done!') ``` However, if you don't have Python up and running, here's a CSV with a smaller version of this data: [testNewQueriesData.csv](https://github.com/elastic/kibana/files/10538537/testNewQueriesData.csv) > **Warning** > When uploading, make sure to update the mappings of the CSV data to the mappings included as part of the Python script above (which you can find as part of the `client.indices.create` call). You'll notice, however, that **none of the CSV documents have a nested field**. Unfortunately, there doesn't seem to be a way to able to ingest nested data through uploading a CSV, so the above data does not include one - in order to test the nested data type, you'd have to add some of your own documents > > Here's a sample nested field document, for your convenience: > ```json > { > "keyword_field": "Russian Federation", > "id": 0, > "boolean_field": true, > "ip_field": "121.149.70.251", > "keyword_text_field": "Michael Foster", > "nested_field": [ > { > "first": "Rachel", > "last": "Wright" > }, > { > "first": "Gary", > "last": "Reyes" > } > ], > "long_keyword_text_field": "Color hotel indicate appear since well sure right yet individual easy often test enough left a usually attention." > } > ``` > ### Testing Notes Because there are now two versions of the queries, thorough testing should be done for both when `search.allow_expensive_queries` is `true` and when it is `false` for every single field type that is currently supported. Use the following call to the cluster settings API to toggle this value back and forth: ```php PUT _cluster/settings { "transient": { "search.allow_expensive_queries": // true or false } } ``` You should pay super special attention to the behaviour that happens when toggling this value from `true` to `false` - for example, consider the following: 1. Ensure `search.allow_expensive_queries` is either `true` or `undefined` 2. Create and save a dashboard with at least one options list control 3. Navigate to the console and set `search.allow_expensive_queries` to `false` - **DO NOT REFRESH** 4. Go back to the dashboard 5. Open up the options list control you created in step 2 6. Fetch a new, uncached request, either by scrolling to the bottom and fetching more (assuming these values aren't already in the cache) or by performing a search with a string you haven't tried before 7. ⚠️ **The options list control _should_ have a fatal error** ⚠️
The Elasticsearch server knows that `search.allow_expensive_queries` is now `false` but, because we only fetch this value on the first load on the client side, it has not yet been updated - this means the options list service still tries to fetch the suggestions using the expensive version of the queries despite the fact that Elasticsearch will now reject this request. The most graceful way to handle this is to simply throw a fatal error. 8. Refreshing the browser will make things sync up again and you should now get the expected results when opening the options list control. ### Flaky Test Runner ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) > **Note** > Technically, it actually does - however, it is due to an [EUI bug](https://github.com/elastic/eui/issues/6565) from adding the group label to the bottom of the list. - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../controls/common/options_list/types.ts | 39 +- src/plugins/controls/common/types.ts | 2 - .../public/__stories__/controls.stories.tsx | 1 - .../component/control_frame_component.tsx | 31 +- .../control_group/editor/control_editor.tsx | 3 - .../editor/data_control_editor_tools.ts | 53 +- .../options_list/components/options_list.scss | 31 +- .../components/options_list_control.test.tsx | 1 + .../components/options_list_control.tsx | 33 +- .../components/options_list_popover.test.tsx | 30 +- .../components/options_list_popover.tsx | 56 +- .../options_list_popover_action_bar.tsx | 152 ++-- .../options_list_popover_footer.tsx | 41 +- .../options_list_popover_sorting_button.tsx | 43 +- .../options_list_popover_suggestions.tsx | 140 ++-- .../components/options_list_popover_title.tsx | 46 ++ .../components/options_list_strings.ts | 32 +- .../embeddable/options_list_embeddable.tsx | 113 +-- .../options_list/options_list_reducers.ts | 20 +- .../controls/public/options_list/types.ts | 14 +- .../public/services/http/http.stub.ts | 1 + .../public/services/http/http_service.ts | 3 +- .../controls/public/services/http/types.ts | 1 + .../options_list/options_list.story.ts | 11 +- .../options_list/options_list_service.ts | 31 +- .../public/services/options_list/types.ts | 11 +- src/plugins/controls/public/types.ts | 2 - ...ons_list_cheap_suggestion_queries.test.ts} | 507 ++++++------- .../options_list_cheap_suggestion_queries.ts | 201 ++++++ .../options_list_cluster_settings_route.ts | 47 ++ ..._list_expensive_suggestion_queries.test.ts | 676 ++++++++++++++++++ ...tions_list_expensive_suggestion_queries.ts | 217 ++++++ .../options_list/options_list_queries.ts | 289 -------- .../options_list_suggestion_query_helpers.ts | 38 + .../options_list_suggestions_route.ts | 41 +- .../options_list_validation_queries.test.ts | 104 +++ .../options_list_validation_queries.ts | 43 ++ .../controls/server/options_list/types.ts | 28 + src/plugins/controls/server/plugin.ts | 3 +- src/plugins/controls/tsconfig.json | 1 + .../embeddable/dashboard_container.tsx | 2 +- .../dashboard_control_group_integration.ts | 11 +- .../lib/embeddables/error_embedabble.scss | 10 - .../lib/embeddables/error_embeddable.scss | 20 + .../lib/embeddables/error_embeddable.tsx | 1 + .../controls/options_list/index.ts | 22 +- ...ptions_list_allow_expensive_queries_off.ts | 96 +++ .../page_objects/dashboard_page_controls.ts | 13 + .../translations/translations/fr-FR.json | 2 - .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - 51 files changed, 2374 insertions(+), 943 deletions(-) create mode 100644 src/plugins/controls/public/options_list/components/options_list_popover_title.tsx rename src/plugins/controls/server/options_list/{options_list_queries.test.ts => options_list_cheap_suggestion_queries.test.ts} (57%) create mode 100644 src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.ts create mode 100644 src/plugins/controls/server/options_list/options_list_cluster_settings_route.ts create mode 100644 src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.test.ts create mode 100644 src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.ts delete mode 100644 src/plugins/controls/server/options_list/options_list_queries.ts create mode 100644 src/plugins/controls/server/options_list/options_list_suggestion_query_helpers.ts create mode 100644 src/plugins/controls/server/options_list/options_list_validation_queries.test.ts create mode 100644 src/plugins/controls/server/options_list/options_list_validation_queries.ts create mode 100644 src/plugins/controls/server/options_list/types.ts delete mode 100644 src/plugins/embeddable/public/lib/embeddables/error_embedabble.scss create mode 100644 src/plugins/embeddable/public/lib/embeddables/error_embeddable.scss create mode 100644 test/functional/apps/dashboard_elements/controls/options_list/options_list_allow_expensive_queries_off.ts diff --git a/src/plugins/controls/common/options_list/types.ts b/src/plugins/controls/common/options_list/types.ts index 1c9555bdc8d90..510dac280fe76 100644 --- a/src/plugins/controls/common/options_list/types.ts +++ b/src/plugins/controls/common/options_list/types.ts @@ -9,8 +9,8 @@ import { FieldSpec, DataView, RuntimeFieldSpec } from '@kbn/data-views-plugin/common'; import type { Filter, Query, BoolQuery, TimeRange } from '@kbn/es-query'; -import { OptionsListSortingType } from './suggestions_sorting'; -import { DataControlInput } from '../types'; +import type { OptionsListSortingType } from './suggestions_sorting'; +import type { DataControlInput } from '../types'; export const OPTIONS_LIST_CONTROL = 'optionsListControl'; @@ -20,20 +20,14 @@ export interface OptionsListEmbeddableInput extends DataControlInput { existsSelected?: boolean; runPastTimeout?: boolean; singleSelect?: boolean; + hideActionBar?: boolean; hideExclude?: boolean; hideExists?: boolean; hideSort?: boolean; - hideActionBar?: boolean; exclude?: boolean; placeholder?: string; } -export type OptionsListField = FieldSpec & { - textFieldName?: string; - parentFieldName?: string; - childFieldName?: string; -}; - export interface OptionsListSuggestions { [key: string]: { doc_count: number }; } @@ -41,13 +35,27 @@ export interface OptionsListSuggestions { /** * The Options list response is returned from the serverside Options List route. */ -export interface OptionsListResponse { - rejected: boolean; +export interface OptionsListSuccessResponse { suggestions: OptionsListSuggestions; - totalCardinality: number; + totalCardinality?: number; // total cardinality will be undefined when `useExpensiveQueries` is `false` invalidSelections?: string[]; } +/** + * The invalid selections are parsed **after** the server returns with the result from the ES client; so, the + * suggestion aggregation parser only returns the suggestions list + the cardinality of the result + */ +export type OptionsListParsedSuggestions = Pick< + OptionsListSuccessResponse, + 'suggestions' | 'totalCardinality' +>; + +export interface OptionsListFailureResponse { + error: 'aborted' | Error; +} + +export type OptionsListResponse = OptionsListSuccessResponse | OptionsListFailureResponse; + /** * The Options list request type taken in by the public Options List service. */ @@ -55,11 +63,12 @@ export type OptionsListRequest = Omit< OptionsListRequestBody, 'filters' | 'fieldName' | 'fieldSpec' | 'textFieldName' > & { + allowExpensiveQueries: boolean; timeRange?: TimeRange; - field: OptionsListField; runPastTimeout?: boolean; dataView: DataView; filters?: Filter[]; + field: FieldSpec; query?: Query; }; @@ -68,13 +77,13 @@ export type OptionsListRequest = Omit< */ export interface OptionsListRequestBody { runtimeFieldMap?: Record; + allowExpensiveQueries: boolean; sort?: OptionsListSortingType; filters?: Array<{ bool: BoolQuery }>; selectedOptions?: string[]; runPastTimeout?: boolean; - parentFieldName?: string; - textFieldName?: string; searchString?: string; fieldSpec?: FieldSpec; fieldName: string; + size: number; } diff --git a/src/plugins/controls/common/types.ts b/src/plugins/controls/common/types.ts index 8f03b82bfaa93..5f37ef2c72871 100644 --- a/src/plugins/controls/common/types.ts +++ b/src/plugins/controls/common/types.ts @@ -30,7 +30,5 @@ export type ControlInput = EmbeddableInput & { export type DataControlInput = ControlInput & { fieldName: string; - parentFieldName?: string; - childFieldName?: string; dataViewId: string; }; diff --git a/src/plugins/controls/public/__stories__/controls.stories.tsx b/src/plugins/controls/public/__stories__/controls.stories.tsx index 6b1f97e39ed7e..4326ce056d118 100644 --- a/src/plugins/controls/public/__stories__/controls.stories.tsx +++ b/src/plugins/controls/public/__stories__/controls.stories.tsx @@ -61,7 +61,6 @@ const storybookStubOptionsListRequest = async ( {} ), totalCardinality: 100, - rejected: false, }), 120 ) diff --git a/src/plugins/controls/public/control_group/component/control_frame_component.tsx b/src/plugins/controls/public/control_group/component/control_frame_component.tsx index 31a63f2fd4eff..eb5cda421381d 100644 --- a/src/plugins/controls/public/control_group/component/control_frame_component.tsx +++ b/src/plugins/controls/public/control_group/component/control_frame_component.tsx @@ -9,15 +9,13 @@ import React, { useEffect, useMemo, useState } from 'react'; import classNames from 'classnames'; import { + EuiButtonEmpty, EuiButtonIcon, EuiFormControlLayout, EuiFormLabel, EuiFormRow, - EuiIcon, - EuiLink, EuiLoadingChart, EuiPopover, - EuiText, EuiToolTip, } from '@elastic/eui'; @@ -40,25 +38,26 @@ interface ControlFrameErrorProps { const ControlFrameError = ({ error }: ControlFrameErrorProps) => { const [isPopoverOpen, setPopoverOpen] = useState(false); const popoverButton = ( - - setPopoverOpen((open) => !open)} - > - - - - + setPopoverOpen((open) => !open)} + className={'errorEmbeddableCompact__button'} + textProps={{ className: 'errorEmbeddableCompact__text' }} + > + + ); return ( setPopoverOpen(false)} > diff --git a/src/plugins/controls/public/control_group/editor/control_editor.tsx b/src/plugins/controls/public/control_group/editor/control_editor.tsx index bff1506280653..785e0e50bd78a 100644 --- a/src/plugins/controls/public/control_group/editor/control_editor.tsx +++ b/src/plugins/controls/public/control_group/editor/control_editor.tsx @@ -199,11 +199,8 @@ export const ControlEditor = ({ selectedFieldName={selectedField} dataView={dataView} onSelectField={(field) => { - const { parentFieldName, childFieldName } = fieldRegistry?.[field.name] ?? {}; onTypeEditorChange({ fieldName: field.name, - ...(parentFieldName && { parentFieldName }), - ...(childFieldName && { childFieldName }), }); const newDefaultTitle = field.displayName ?? field.name; setDefaultTitle(newDefaultTitle); diff --git a/src/plugins/controls/public/control_group/editor/data_control_editor_tools.ts b/src/plugins/controls/public/control_group/editor/data_control_editor_tools.ts index 4344891280ce6..8cfd3cebe5dac 100644 --- a/src/plugins/controls/public/control_group/editor/data_control_editor_tools.ts +++ b/src/plugins/controls/public/control_group/editor/data_control_editor_tools.ts @@ -8,11 +8,10 @@ import { memoize } from 'lodash'; -import { IFieldSubTypeMulti } from '@kbn/es-query'; import { DataView } from '@kbn/data-views-plugin/common'; import { pluginServices } from '../../services'; -import { DataControlFieldRegistry, IEditableControlFactory } from '../../types'; +import { DataControlField, DataControlFieldRegistry, IEditableControlFactory } from '../../types'; export const getDataControlFieldRegistry = memoize( async (dataView: DataView) => { @@ -21,50 +20,30 @@ export const getDataControlFieldRegistry = memoize( (dataView: DataView) => [dataView.id, JSON.stringify(dataView.fields.getAll())].join('|') ); -const doubleLinkFields = (dataView: DataView) => { - // double link the parent-child relationship specifically for case-sensitivity support for options lists - const fieldRegistry: DataControlFieldRegistry = {}; - - for (const field of dataView.fields.getAll()) { - if (!fieldRegistry[field.name]) { - fieldRegistry[field.name] = { field, compatibleControlTypes: [] }; - } - - const parentFieldName = (field.subType as IFieldSubTypeMulti)?.multi?.parent; - if (parentFieldName) { - fieldRegistry[field.name].parentFieldName = parentFieldName; - - const parentField = dataView.getFieldByName(parentFieldName); - if (!fieldRegistry[parentFieldName] && parentField) { - fieldRegistry[parentFieldName] = { field: parentField, compatibleControlTypes: [] }; - } - fieldRegistry[parentFieldName].childFieldName = field.name; - } - } - return fieldRegistry; -}; - const loadFieldRegistryFromDataView = async ( dataView: DataView ): Promise => { const { controls: { getControlTypes, getControlFactory }, } = pluginServices.getServices(); - const newFieldRegistry: DataControlFieldRegistry = doubleLinkFields(dataView); + const controlFactories = getControlTypes().map( (controlType) => getControlFactory(controlType) as IEditableControlFactory ); - dataView.fields.map((dataViewField) => { - for (const factory of controlFactories) { - if (factory.isFieldCompatible) { - factory.isFieldCompatible(newFieldRegistry[dataViewField.name]); + const fieldRegistry: DataControlFieldRegistry = dataView.fields + .getAll() + .reduce((registry, field) => { + const test: DataControlField = { field, compatibleControlTypes: [] }; + for (const factory of controlFactories) { + if (factory.isFieldCompatible) { + factory.isFieldCompatible(test); + } } - } - - if (newFieldRegistry[dataViewField.name]?.compatibleControlTypes.length === 0) { - delete newFieldRegistry[dataViewField.name]; - } - }); + if (test.compatibleControlTypes.length === 0) { + return { ...registry }; + } + return { ...registry, [field.name]: test }; + }, {}); - return newFieldRegistry; + return fieldRegistry; }; diff --git a/src/plugins/controls/public/options_list/components/options_list.scss b/src/plugins/controls/public/options_list/components/options_list.scss index e88208ee4c623..ad042916fff6e 100644 --- a/src/plugins/controls/public/options_list/components/options_list.scss +++ b/src/plugins/controls/public/options_list/components/options_list.scss @@ -9,8 +9,18 @@ .optionsList__actions { padding: $euiSizeS; + padding-bottom: 0; border-bottom: $euiBorderThin; border-color: darken($euiColorLightestShade, 2%); + + .optionsList__actionsRow { + margin: ($euiSizeS / 2) 0 !important; + + .optionsList__actionBarDivider { + height: $euiSize; + border-right: $euiBorderThin; + } + } } .optionsList__popoverTitle { @@ -30,13 +40,17 @@ font-style: italic; } +.optionsList__loadMore { + font-style: italic; +} + .optionsList__negateLabel { font-weight: bold; font-size: $euiSizeM; color: $euiColorDanger; } -.optionsList__ignoredBadge { +.optionsList__actionBarFirstBadge { margin-left: $euiSizeS; } @@ -86,3 +100,18 @@ .optionsList--sortPopover { width: $euiSizeXL * 7; } + +.optionslist--loadingMoreGroupLabel { + text-align: center; + padding: $euiSizeM; + font-style: italic; + height: $euiSizeXXL !important; +} + +.optionslist--endOfOptionsGroupLabel { + text-align: center; + font-size: $euiSizeM; + height: auto !important; + color: $euiTextSubduedColor; + padding: $euiSizeM; +} diff --git a/src/plugins/controls/public/options_list/components/options_list_control.test.tsx b/src/plugins/controls/public/options_list/components/options_list_control.test.tsx index a4d5028f0f7be..7fe1cd2f7aa78 100644 --- a/src/plugins/controls/public/options_list/components/options_list_control.test.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_control.test.tsx @@ -20,6 +20,7 @@ import { BehaviorSubject } from 'rxjs'; describe('Options list control', () => { const defaultProps = { typeaheadSubject: new BehaviorSubject(''), + loadMoreSubject: new BehaviorSubject(10), }; interface MountOptions { diff --git a/src/plugins/controls/public/options_list/components/options_list_control.tsx b/src/plugins/controls/public/options_list/components/options_list_control.tsx index 98f545718efc6..7906d77730f0d 100644 --- a/src/plugins/controls/public/options_list/components/options_list_control.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_control.tsx @@ -17,20 +17,24 @@ import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public' import { OptionsListStrings } from './options_list_strings'; import { OptionsListPopover } from './options_list_popover'; import { optionsListReducers } from '../options_list_reducers'; -import { OptionsListReduxState } from '../types'; +import { MAX_OPTIONS_LIST_REQUEST_SIZE, OptionsListReduxState } from '../types'; import './options_list.scss'; -export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Subject }) => { - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - +export const OptionsListControl = ({ + typeaheadSubject, + loadMoreSubject, +}: { + typeaheadSubject: Subject; + loadMoreSubject: Subject; +}) => { const resizeRef = useRef(null); const dimensions = useResizeObserver(resizeRef.current); // Redux embeddable Context const { useEmbeddableDispatch, - actions: { replaceSelection, setSearchString }, + actions: { replaceSelection, setSearchString, setPopoverOpen }, useEmbeddableSelector: select, } = useReduxEmbeddableContext(); const dispatch = useEmbeddableDispatch(); @@ -38,6 +42,7 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub // Select current state from Redux using multiple selectors to avoid rerenders. const invalidSelections = select((state) => state.componentState.invalidSelections); const validSelections = select((state) => state.componentState.validSelections); + const isPopoverOpen = select((state) => state.componentState.popoverOpen); const selectedOptions = select((state) => state.explicitInput.selectedOptions); const existsSelected = select((state) => state.explicitInput.existsSelected); @@ -51,6 +56,12 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub const loading = select((state) => state.output.loading); + useEffect(() => { + return () => { + dispatch(setPopoverOpen(false)); // on unmount, close the popover + }; + }, [dispatch, setPopoverOpen]); + // debounce loading state so loading doesn't flash when user types const [debouncedLoading, setDebouncedLoading] = useState(true); const debounceSetLoading = useMemo( @@ -77,6 +88,13 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub [typeaheadSubject, dispatch, setSearchString] ); + const loadMoreSuggestions = useCallback( + (cardinality: number) => { + loadMoreSubject.next(Math.min(cardinality, MAX_OPTIONS_LIST_REQUEST_SIZE)); + }, + [loadMoreSubject] + ); + const { hasSelections, selectionDisplayNode, validSelectionsCount } = useMemo(() => { return { hasSelections: !isEmpty(validSelections) || !isEmpty(invalidSelections), @@ -123,7 +141,7 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub 'optionsList--filterBtnPlaceholder': !hasSelections, })} data-test-subj={`optionsList-control-${id}`} - onClick={() => setIsPopoverOpen((openState) => !openState)} + onClick={() => dispatch(setPopoverOpen(!isPopoverOpen))} isSelected={isPopoverOpen} numActiveFilters={validSelectionsCount} hasActiveFilters={Boolean(validSelectionsCount)} @@ -149,7 +167,7 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub panelPaddingSize="none" anchorPosition="downCenter" className="optionsList__popoverOverride" - closePopover={() => setIsPopoverOpen(false)} + closePopover={() => dispatch(setPopoverOpen(false))} anchorClassName="optionsList__anchorOverride" aria-label={OptionsListStrings.popover.getAriaLabel(fieldName)} > @@ -157,6 +175,7 @@ export const OptionsListControl = ({ typeaheadSubject }: { typeaheadSubject: Sub width={dimensions.width} isLoading={debouncedLoading} updateSearchString={updateSearchString} + loadMoreSuggestions={loadMoreSuggestions} /> diff --git a/src/plugins/controls/public/options_list/components/options_list_popover.test.tsx b/src/plugins/controls/public/options_list/components/options_list_popover.test.tsx index a8504aba372c8..4b40414974b67 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover.test.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover.test.tsx @@ -11,11 +11,11 @@ import { ReactWrapper } from 'enzyme'; import { mountWithIntl } from '@kbn/test-jest-helpers'; import { findTestSubject } from '@elastic/eui/lib/test'; +import { FieldSpec } from '@kbn/data-views-plugin/common'; import { OptionsListPopover, OptionsListPopoverProps } from './options_list_popover'; import { OptionsListComponentState, OptionsListReduxState } from '../types'; import { mockOptionsListReduxEmbeddableTools } from '../../../common/mocks'; -import { OptionsListField } from '../../../common/options_list/types'; import { ControlOutput, OptionsListEmbeddableInput } from '../..'; describe('Options list popover', () => { @@ -23,6 +23,7 @@ describe('Options list popover', () => { width: 500, isLoading: false, updateSearchString: jest.fn(), + loadMoreSuggestions: jest.fn(), }; interface MountOptions { @@ -63,7 +64,7 @@ describe('Options list popover', () => { // the div cannot be smaller than 301 pixels wide popover = await mountComponent({ popoverProps: { width: 300 } }); popoverDiv = findTestSubject(popover, 'optionsList-control-available-options'); - expect(popoverDiv.getDOMNode().getAttribute('style')).toBe(null); + expect(popoverDiv.getDOMNode().getAttribute('style')).toBe('width: 100%; height: 100%;'); }); test('no available options', async () => { @@ -237,7 +238,7 @@ describe('Options list popover', () => { test('when sorting suggestions, show both sorting types for keyword field', async () => { const popover = await mountComponent({ componentState: { - field: { name: 'Test keyword field', type: 'keyword' } as OptionsListField, + field: { name: 'Test keyword field', type: 'keyword' } as FieldSpec, }, }); const sortButton = findTestSubject(popover, 'optionsListControl__sortingOptionsButton'); @@ -252,7 +253,7 @@ describe('Options list popover', () => { const popover = await mountComponent({ explicitInput: { sort: { by: '_key', direction: 'asc' } }, componentState: { - field: { name: 'Test keyword field', type: 'keyword' } as OptionsListField, + field: { name: 'Test keyword field', type: 'keyword' } as FieldSpec, }, }); const sortButton = findTestSubject(popover, 'optionsListControl__sortingOptionsButton'); @@ -270,7 +271,7 @@ describe('Options list popover', () => { test('when sorting suggestions, only show document count sorting for IP fields', async () => { const popover = await mountComponent({ - componentState: { field: { name: 'Test IP field', type: 'ip' } as OptionsListField }, + componentState: { field: { name: 'Test IP field', type: 'ip' } as FieldSpec }, }); const sortButton = findTestSubject(popover, 'optionsListControl__sortingOptionsButton'); sortButton.simulate('click'); @@ -280,6 +281,25 @@ describe('Options list popover', () => { expect(optionsText).toEqual(['By document count - Checked option.']); }); + test('ensure warning icon does not show up when testAllowExpensiveQueries = true/undefined', async () => { + const popover = await mountComponent({ + componentState: { field: { name: 'Test keyword field', type: 'keyword' } as FieldSpec }, + }); + const warning = findTestSubject(popover, 'optionsList-allow-expensive-queries-warning'); + expect(warning).toEqual({}); + }); + + test('ensure warning icon shows up when testAllowExpensiveQueries = false', async () => { + const popover = await mountComponent({ + componentState: { + field: { name: 'Test keyword field', type: 'keyword' } as FieldSpec, + allowExpensiveQueries: false, + }, + }); + const warning = findTestSubject(popover, 'optionsList-allow-expensive-queries-warning'); + expect(warning.getDOMNode()).toBeInstanceOf(HTMLDivElement); + }); + describe('Test advanced settings', () => { const ensureComponentIsHidden = async ({ explicitInput, diff --git a/src/plugins/controls/public/options_list/components/options_list_popover.tsx b/src/plugins/controls/public/options_list/components/options_list_popover.tsx index b5562c43f3be2..df5f3d00730d0 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover.tsx @@ -9,12 +9,12 @@ import React, { useState } from 'react'; import { isEmpty } from 'lodash'; -import { EuiPopoverTitle } from '@elastic/eui'; import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; import { OptionsListReduxState } from '../types'; import { OptionsListStrings } from './options_list_strings'; import { optionsListReducers } from '../options_list_reducers'; +import { OptionsListPopoverTitle } from './options_list_popover_title'; import { OptionsListPopoverFooter } from './options_list_popover_footer'; import { OptionsListPopoverActionBar } from './options_list_popover_action_bar'; import { OptionsListPopoverSuggestions } from './options_list_popover_suggestions'; @@ -23,6 +23,7 @@ import { OptionsListPopoverInvalidSelections } from './options_list_popover_inva export interface OptionsListPopoverProps { width: number; isLoading: boolean; + loadMoreSuggestions: (cardinality: number) => void; updateSearchString: (newSearchString: string) => void; } @@ -30,6 +31,7 @@ export const OptionsListPopover = ({ width, isLoading, updateSearchString, + loadMoreSuggestions, }: OptionsListPopoverProps) => { // Redux embeddable container Context const { useEmbeddableSelector: select } = useReduxEmbeddableContext< @@ -42,39 +44,45 @@ export const OptionsListPopover = ({ const availableOptions = select((state) => state.componentState.availableOptions); const field = select((state) => state.componentState.field); - const hideExclude = select((state) => state.explicitInput.hideExclude); const hideActionBar = select((state) => state.explicitInput.hideActionBar); + const hideExclude = select((state) => state.explicitInput.hideExclude); const fieldName = select((state) => state.explicitInput.fieldName); - const title = select((state) => state.explicitInput.title); const id = select((state) => state.explicitInput.id); const [showOnlySelected, setShowOnlySelected] = useState(false); return ( -
- {title} - {field?.type !== 'boolean' && !hideActionBar && ( - - )} + <>
- - {!showOnlySelected && invalidSelections && !isEmpty(invalidSelections) && ( - + + + {field?.type !== 'boolean' && !hideActionBar && ( + )} +
+ + {!showOnlySelected && invalidSelections && !isEmpty(invalidSelections) && ( + + )} +
+ {!hideExclude && }
- {!hideExclude && } -
+ ); }; diff --git a/src/plugins/controls/public/options_list/components/options_list_popover_action_bar.tsx b/src/plugins/controls/public/options_list/components/options_list_popover_action_bar.tsx index 375a2a2058692..fc3c1cfdfd993 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover_action_bar.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover_action_bar.tsx @@ -15,7 +15,7 @@ import { EuiFlexItem, EuiFormRow, EuiToolTip, - EuiBadge, + EuiText, } from '@elastic/eui'; import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; @@ -26,8 +26,8 @@ import { OptionsListPopoverSortingButton } from './options_list_popover_sorting_ interface OptionsListPopoverProps { showOnlySelected: boolean; - setShowOnlySelected: (value: boolean) => void; updateSearchString: (newSearchString: string) => void; + setShowOnlySelected: (value: boolean) => void; } export const OptionsListPopoverActionBar = ({ @@ -44,23 +44,17 @@ export const OptionsListPopoverActionBar = ({ const dispatch = useEmbeddableDispatch(); // Select current state from Redux using multiple selectors to avoid rerenders. + const allowExpensiveQueries = select((state) => state.componentState.allowExpensiveQueries); const invalidSelections = select((state) => state.componentState.invalidSelections); - const totalCardinality = select((state) => state.componentState.totalCardinality); + const totalCardinality = select((state) => state.componentState.totalCardinality) ?? 0; const searchString = select((state) => state.componentState.searchString); - const hideSort = select((state) => state.explicitInput.hideSort); return (
- - + + updateSearchString(event.target.value)} value={searchString.value} data-test-subj="optionsList-control-search-input" - placeholder={ - totalCardinality - ? OptionsListStrings.popover.getTotalCardinalityPlaceholder(totalCardinality) - : undefined - } + placeholder={OptionsListStrings.popover.getSearchPlaceholder()} autoFocus={true} /> - - {(invalidSelections?.length ?? 0) > 0 && ( - - - {invalidSelections?.length} - - - )} - {!hideSort && ( )} - - - setShowOnlySelected(!showOnlySelected)} - data-test-subj="optionsList-control-show-only-selected" - aria-label={ - showOnlySelected - ? OptionsListStrings.popover.getAllOptionsButtonTitle() - : OptionsListStrings.popover.getSelectedOptionsButtonTitle() - } - /> - - - - + + + + {allowExpensiveQueries && ( + + + {OptionsListStrings.popover.getCardinalityLabel(totalCardinality)} + + + )} + {invalidSelections && invalidSelections.length > 0 && ( + <> + {allowExpensiveQueries && ( + +
+ + )} + + + {OptionsListStrings.popover.getInvalidSelectionsLabel(invalidSelections.length)} + + + + )} + + - dispatch(clearSelections({}))} - data-test-subj="optionsList-control-clear-all-selections" - aria-label={OptionsListStrings.popover.getClearAllSelectionsButtonTitle()} - /> - + + + setShowOnlySelected(!showOnlySelected)} + data-test-subj="optionsList-control-show-only-selected" + aria-label={ + showOnlySelected + ? OptionsListStrings.popover.getAllOptionsButtonTitle() + : OptionsListStrings.popover.getSelectedOptionsButtonTitle() + } + /> + + + + + dispatch(clearSelections({}))} + data-test-subj="optionsList-control-clear-all-selections" + aria-label={OptionsListStrings.popover.getClearAllSelectionsButtonTitle()} + /> + + + diff --git a/src/plugins/controls/public/options_list/components/options_list_popover_footer.tsx b/src/plugins/controls/public/options_list/components/options_list_popover_footer.tsx index 8a51a33a31ba0..d04e389f8ec99 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover_footer.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover_footer.tsx @@ -7,7 +7,13 @@ */ import React from 'react'; -import { EuiPopoverFooter, EuiButtonGroup, useEuiBackgroundColor } from '@elastic/eui'; +import { + useEuiBackgroundColor, + useEuiPaddingSize, + EuiPopoverFooter, + EuiButtonGroup, + EuiProgress, +} from '@elastic/eui'; import { css } from '@emotion/react'; import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; @@ -26,7 +32,7 @@ const aggregationToggleButtons = [ }, ]; -export const OptionsListPopoverFooter = () => { +export const OptionsListPopoverFooter = ({ isLoading }: { isLoading: boolean }) => { // Redux embeddable container Context const { useEmbeddableDispatch, @@ -41,19 +47,32 @@ export const OptionsListPopoverFooter = () => { return ( <> - dispatch(setExclude(optionId === 'optionsList__excludeResults'))} - buttonSize="compressed" - data-test-subj="optionsList__includeExcludeButtonGroup" - /> + {isLoading && ( +
+ +
+ )} +
+ + dispatch(setExclude(optionId === 'optionsList__excludeResults')) + } + buttonSize="compressed" + data-test-subj="optionsList__includeExcludeButtonGroup" + /> +
); diff --git a/src/plugins/controls/public/options_list/components/options_list_popover_sorting_button.tsx b/src/plugins/controls/public/options_list/components/options_list_popover_sorting_button.tsx index 91482e6f54668..9132fee3dc5dc 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover_sorting_button.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover_sorting_button.tsx @@ -12,14 +12,14 @@ import { EuiButtonGroupOptionProps, EuiSelectableOption, EuiPopoverTitle, + EuiButtonEmpty, EuiButtonGroup, - EuiButtonIcon, EuiSelectable, EuiFlexGroup, EuiFlexItem, - EuiToolTip, EuiPopover, Direction, + EuiToolTip, } from '@elastic/eui'; import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; @@ -91,25 +91,32 @@ export const OptionsListPopoverSortingButton = ({ } }; + const SortButton = () => ( + setIsSortingPopoverOpen(!isSortingPopoverOpen)} + className="euiFilterGroup" // this gives the button a nice border + aria-label={OptionsListStrings.popover.getSortPopoverDescription()} + > + {OptionsListStrings.popover.getSortPopoverTitle()} + + ); + return ( - setIsSortingPopoverOpen(!isSortingPopoverOpen)} - aria-label={OptionsListStrings.popover.getSortPopoverDescription()} - /> - + showOnlySelected ? ( + + + + ) : ( + + ) } panelPaddingSize="none" isOpen={isSortingPopoverOpen} diff --git a/src/plugins/controls/public/options_list/components/options_list_popover_suggestions.tsx b/src/plugins/controls/public/options_list/components/options_list_popover_suggestions.tsx index 8bd8e361e7081..fd0e7668c82fa 100644 --- a/src/plugins/controls/public/options_list/components/options_list_popover_suggestions.tsx +++ b/src/plugins/controls/public/options_list/components/options_list_popover_suggestions.tsx @@ -6,26 +6,27 @@ * Side Public License, v 1. */ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { EuiLoadingSpinner, EuiSelectable, EuiSpacer } from '@elastic/eui'; +import { euiThemeVars } from '@kbn/ui-theme'; +import { EuiSelectable } from '@elastic/eui'; import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; import { EuiSelectableOption } from '@elastic/eui/src/components/selectable/selectable_option'; -import { OptionsListReduxState } from '../types'; import { OptionsListStrings } from './options_list_strings'; import { optionsListReducers } from '../options_list_reducers'; +import { MAX_OPTIONS_LIST_REQUEST_SIZE, OptionsListReduxState } from '../types'; import { OptionsListPopoverEmptyMessage } from './options_list_popover_empty_message'; import { OptionsListPopoverSuggestionBadge } from './options_list_popover_suggestion_badge'; interface OptionsListPopoverSuggestionsProps { - isLoading: boolean; showOnlySelected: boolean; + loadMoreSuggestions: (cardinality: number) => void; } export const OptionsListPopoverSuggestions = ({ - isLoading, showOnlySelected, + loadMoreSuggestions, }: OptionsListPopoverSuggestionsProps) => { // Redux embeddable container Context const { @@ -38,12 +39,27 @@ export const OptionsListPopoverSuggestions = ({ // Select current state from Redux using multiple selectors to avoid rerenders. const invalidSelections = select((state) => state.componentState.invalidSelections); const availableOptions = select((state) => state.componentState.availableOptions); + const totalCardinality = select((state) => state.componentState.totalCardinality); + const searchString = select((state) => state.componentState.searchString); const selectedOptions = select((state) => state.explicitInput.selectedOptions); const existsSelected = select((state) => state.explicitInput.existsSelected); const singleSelect = select((state) => state.explicitInput.singleSelect); const hideExists = select((state) => state.explicitInput.hideExists); + const isLoading = select((state) => state.output.loading) ?? false; const fieldName = select((state) => state.explicitInput.fieldName); + const sort = select((state) => state.explicitInput.sort); + + const listRef = useRef(null); + + const canLoadMoreSuggestions = useMemo( + () => + totalCardinality + ? Object.keys(availableOptions ?? {}).length < + Math.min(totalCardinality, MAX_OPTIONS_LIST_REQUEST_SIZE) + : false, + [availableOptions, totalCardinality] + ); // track selectedOptions and invalidSelections in sets for more efficient lookup const selectedOptionsSet = useMemo(() => new Set(selectedOptions), [selectedOptions]); @@ -51,7 +67,6 @@ export const OptionsListPopoverSuggestions = ({ () => new Set(invalidSelections), [invalidSelections] ); - const suggestions = useMemo(() => { return showOnlySelected ? selectedOptions : Object.keys(availableOptions ?? {}); }, [availableOptions, selectedOptions, showOnlySelected]); @@ -87,10 +102,23 @@ export const OptionsListPopoverSuggestions = ({ ) : undefined, }; }); - const suggestionsSelectableOptions = existsSelectableOption - ? [existsSelectableOption, ...options] - : options; - setSelectableOptions(suggestionsSelectableOptions); + + if (canLoadMoreSuggestions) { + options.push({ + key: 'loading-option', + className: 'optionslist--loadingMoreGroupLabel', + label: OptionsListStrings.popover.getLoadingMoreMessage(), + isGroupLabel: true, + }); + } else if (options.length === MAX_OPTIONS_LIST_REQUEST_SIZE) { + options.push({ + key: 'no-more-option', + className: 'optionslist--endOfOptionsGroupLabel', + label: OptionsListStrings.popover.getAtEndOfOptionsMessage(), + isGroupLabel: true, + }); + } + setSelectableOptions(existsSelectableOption ? [existsSelectableOption, ...options] : options); }, [ suggestions, availableOptions, @@ -98,42 +126,66 @@ export const OptionsListPopoverSuggestions = ({ selectedOptionsSet, invalidSelectionsSet, existsSelectableOption, + canLoadMoreSuggestions, ]); + const loadMoreOptions = useCallback(() => { + const listbox = listRef.current?.querySelector('.euiSelectableList__list'); + if (!listbox) return; + + const { scrollTop, scrollHeight, clientHeight } = listbox; + if (scrollTop + clientHeight >= scrollHeight - parseInt(euiThemeVars.euiSizeXXL, 10)) { + // reached the "bottom" of the list, where euiSizeXXL acts as a "margin of error" so that the user doesn't + // have to scroll **all the way** to the bottom in order to load more options + loadMoreSuggestions(totalCardinality ?? MAX_OPTIONS_LIST_REQUEST_SIZE); + } + }, [loadMoreSuggestions, totalCardinality]); + + useEffect(() => { + const container = listRef.current; + if (!isLoading && canLoadMoreSuggestions) { + container?.addEventListener('scroll', loadMoreOptions, true); + return () => { + container?.removeEventListener('scroll', loadMoreOptions, true); + }; + } + }, [loadMoreOptions, isLoading, canLoadMoreSuggestions]); + + useEffect(() => { + // scroll back to the top when changing the sorting or the search string + const listbox = listRef.current?.querySelector('.euiSelectableList__list'); + listbox?.scrollTo({ top: 0 }); + }, [sort, searchString]); + return ( - - - - {OptionsListStrings.popover.getLoadingMessage()} - - } - options={selectableOptions} - listProps={{ onFocusBadge: false }} - aria-label={OptionsListStrings.popover.getSuggestionsAriaLabel( - fieldName, - selectableOptions.length - )} - emptyMessage={} - onChange={(newSuggestions, _, changedOption) => { - setSelectableOptions(newSuggestions); - - const key = changedOption.key ?? changedOption.label; - // the order of these checks matters, so be careful if rearranging them - if (key === 'exists-option') { - dispatch(selectExists(!Boolean(existsSelected))); - } else if (showOnlySelected || selectedOptionsSet.has(key)) { - dispatch(deselectOption(key)); - } else if (singleSelect) { - dispatch(replaceSelection(key)); - } else { - dispatch(selectOption(key)); - } - }} - > - {(list) => list} - + <> +
+ } + onChange={(newSuggestions, _, changedOption) => { + const key = changedOption.key ?? changedOption.label; + setSelectableOptions(newSuggestions); + // the order of these checks matters, so be careful if rearranging them + if (key === 'exists-option') { + dispatch(selectExists(!Boolean(existsSelected))); + } else if (showOnlySelected || selectedOptionsSet.has(key)) { + dispatch(deselectOption(key)); + } else if (singleSelect) { + dispatch(replaceSelection(key)); + } else { + dispatch(selectOption(key)); + } + }} + > + {(list) => list} + +
+ ); }; diff --git a/src/plugins/controls/public/options_list/components/options_list_popover_title.tsx b/src/plugins/controls/public/options_list/components/options_list_popover_title.tsx new file mode 100644 index 0000000000000..40361260e56c0 --- /dev/null +++ b/src/plugins/controls/public/options_list/components/options_list_popover_title.tsx @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React from 'react'; + +import { EuiFlexGroup, EuiFlexItem, EuiPopoverTitle, EuiIconTip } from '@elastic/eui'; +import { useReduxEmbeddableContext } from '@kbn/presentation-util-plugin/public'; + +import { OptionsListReduxState } from '../types'; +import { OptionsListStrings } from './options_list_strings'; +import { optionsListReducers } from '../options_list_reducers'; + +export const OptionsListPopoverTitle = () => { + // Redux embeddable container Context + const { useEmbeddableSelector: select } = useReduxEmbeddableContext< + OptionsListReduxState, + typeof optionsListReducers + >(); + + // Select current state from Redux using multiple selectors to avoid rerenders. + const allowExpensiveQueries = select((state) => state.componentState.allowExpensiveQueries); + const title = select((state) => state.explicitInput.title); + + return ( + + + {title} + {!allowExpensiveQueries && ( + + + + )} + + + ); +}; diff --git a/src/plugins/controls/public/options_list/components/options_list_strings.ts b/src/plugins/controls/public/options_list/components/options_list_strings.ts index bef8a2cbc26ff..f67676103a92d 100644 --- a/src/plugins/controls/public/options_list/components/options_list_strings.ts +++ b/src/plugins/controls/public/options_list/components/options_list_strings.ts @@ -54,9 +54,19 @@ export const OptionsListStrings = { 'Available {optionCount, plural, one {option} other {options}} for {fieldName}', values: { fieldName, optionCount }, }), - getLoadingMessage: () => - i18n.translate('controls.optionsList.popover.loading', { - defaultMessage: 'Loading options', + getAllowExpensiveQueriesWarning: () => + i18n.translate('controls.optionsList.popover.allowExpensiveQueriesWarning', { + defaultMessage: + 'The cluster setting to allow expensive queries is off, so some features are disabled.', + }), + getLoadingMoreMessage: () => + i18n.translate('controls.optionsList.popover.loadingMore', { + defaultMessage: 'Loading more options...', + }), + getAtEndOfOptionsMessage: () => + i18n.translate('controls.optionsList.popover.endOfOptions', { + defaultMessage: + 'The top 1,000 available options are displayed. View more options by searching for the name.', }), getEmptyMessage: () => i18n.translate('controls.optionsList.popover.empty', { @@ -78,10 +88,14 @@ export const OptionsListStrings = { i18n.translate('controls.optionsList.popover.clearAllSelectionsTitle', { defaultMessage: 'Clear selections', }), - getTotalCardinalityPlaceholder: (totalOptions: number) => - i18n.translate('controls.optionsList.popover.cardinalityPlaceholder', { + getSearchPlaceholder: () => + i18n.translate('controls.optionsList.popover.searchPlaceholder', { + defaultMessage: 'Search', + }), + getCardinalityLabel: (totalOptions: number) => + i18n.translate('controls.optionsList.popover.cardinalityLabel', { defaultMessage: - 'Search {totalOptions} available {totalOptions, plural, one {option} other {options}}', + '{totalOptions, number} {totalOptions, plural, one {option} other {options}}', values: { totalOptions }, }), getInvalidSelectionsSectionAriaLabel: (fieldName: string, invalidSelectionCount: number) => @@ -96,10 +110,10 @@ export const OptionsListStrings = { 'Ignored {invalidSelectionCount, plural, one {selection} other {selections}}', values: { invalidSelectionCount }, }), - getInvalidSelectionsTooltip: (selectedOptions: number) => - i18n.translate('controls.optionsList.popover.invalidSelectionsTooltip', { + getInvalidSelectionsLabel: (selectedOptions: number) => + i18n.translate('controls.optionsList.popover.invalidSelectionsLabel', { defaultMessage: - '{selectedOptions} selected {selectedOptions, plural, one {option} other {options}} {selectedOptions, plural, one {is} other {are}} ignored because {selectedOptions, plural, one {it is} other {they are}} no longer in the data.', + '{selectedOptions} {selectedOptions, plural, one {selection} other {selections}} ignored', values: { selectedOptions }, }), getInvalidSelectionScreenReaderText: () => diff --git a/src/plugins/controls/public/options_list/embeddable/options_list_embeddable.tsx b/src/plugins/controls/public/options_list/embeddable/options_list_embeddable.tsx index 8d8fb505b5cdd..2a7f7e116c40f 100644 --- a/src/plugins/controls/public/options_list/embeddable/options_list_embeddable.tsx +++ b/src/plugins/controls/public/options_list/embeddable/options_list_embeddable.tsx @@ -24,11 +24,11 @@ import { buildExistsFilter, } from '@kbn/es-query'; import { ReduxEmbeddableTools, ReduxEmbeddablePackage } from '@kbn/presentation-util-plugin/public'; -import { DataView } from '@kbn/data-views-plugin/public'; +import { DataView, FieldSpec } from '@kbn/data-views-plugin/public'; import { Embeddable, IContainer } from '@kbn/embeddable-plugin/public'; import { KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; -import { OptionsListReduxState } from '../types'; +import { MIN_OPTIONS_LIST_REQUEST_SIZE, OptionsListReduxState } from '../types'; import { pluginServices } from '../../services'; import { ControlInput, @@ -40,7 +40,6 @@ import { getDefaultComponentState, optionsListReducers } from '../options_list_r import { OptionsListControl } from '../components/options_list_control'; import { ControlsDataViewsService } from '../../services/data_views/types'; import { ControlsOptionsListService } from '../../services/options_list/types'; -import { OptionsListField } from '../../../common/options_list/types'; const diffDataFetchProps = ( last?: OptionsListDataFetchProps, @@ -76,9 +75,10 @@ export class OptionsListEmbeddable extends Embeddable = new Subject(); + private loadMoreSubject: Subject = new Subject(); private abortController?: AbortController; private dataView?: DataView; - private field?: OptionsListField; + private field?: FieldSpec; private reduxEmbeddableTools: ReduxEmbeddableTools< OptionsListReduxState, @@ -98,6 +98,7 @@ export class OptionsListEmbeddable extends Embeddable(); + this.loadMoreSubject = new Subject(); // build redux embeddable tools this.reduxEmbeddableTools = reduxEmbeddablePackage.createTools< @@ -115,6 +116,13 @@ export class OptionsListEmbeddable extends Embeddable { const { selectedOptions: initialSelectedOptions } = this.getInput(); if (!initialSelectedOptions) this.setInitializationFinished(); + + const { + actions: { setAllowExpensiveQueries }, + dispatch, + } = this.reduxEmbeddableTools; + dispatch(setAllowExpensiveQueries(await this.optionsListService.getAllowExpensiveQueries())); + this.runOptionsListQuery().then(async () => { if (initialSelectedOptions) { await this.buildFilter(); @@ -144,12 +152,21 @@ export class OptionsListEmbeddable extends Embeddable { + this.runOptionsListQuery(); + }) + ); + // fetch more options when reaching the bottom of the available options + this.subscriptions.add( + loadMorePipe.subscribe((size) => { + this.runOptionsListQuery(size); + }) ); /** @@ -203,7 +220,7 @@ export class OptionsListEmbeddable extends Embeddable => { const { dispatch, @@ -212,7 +229,7 @@ export class OptionsListEmbeddable extends Embeddable { + private runOptionsListQuery = async (size: number = MIN_OPTIONS_LIST_REQUEST_SIZE) => { const { dispatch, getState, actions: { setLoading, publishFilters, setSearchString, updateQueryResults }, } = this.reduxEmbeddableTools; - const previousFieldName = this.field?.name; const { dataView, field } = await this.getCurrentDataViewAndField(); if (!dataView || !field) return; @@ -284,7 +286,7 @@ export class OptionsListEmbeddable extends Embeddable { + const { + dispatch, + actions: { setPopoverOpen, setLoading }, + } = this.reduxEmbeddableTools; + batch(() => { + dispatch(setLoading(false)); + dispatch(setPopoverOpen(false)); + }); + super.onFatalError(e); + }; + public destroy = () => { super.destroy(); this.abortController?.abort(); this.subscriptions.unsubscribe(); this.reduxEmbeddableTools.cleanup(); + if (this.node) ReactDOM.unmountComponentAtNode(this.node); }; public render = (node: HTMLElement) => { @@ -427,7 +449,10 @@ export class OptionsListEmbeddable extends Embeddable - + , node diff --git a/src/plugins/controls/public/options_list/options_list_reducers.ts b/src/plugins/controls/public/options_list/options_list_reducers.ts index e2d7388c7f81e..f50ab9bd6f7f7 100644 --- a/src/plugins/controls/public/options_list/options_list_reducers.ts +++ b/src/plugins/controls/public/options_list/options_list_reducers.ts @@ -9,9 +9,9 @@ import { PayloadAction } from '@reduxjs/toolkit'; import { WritableDraft } from 'immer/dist/types/types-external'; import { Filter } from '@kbn/es-query'; +import { FieldSpec } from '@kbn/data-views-plugin/common'; import { OptionsListReduxState, OptionsListComponentState } from './types'; -import { OptionsListField } from '../../common/options_list/types'; import { getIpRangeQuery } from '../../common/options_list/ip_search'; import { OPTIONS_LIST_DEFAULT_SORT, @@ -19,6 +19,8 @@ import { } from '../../common/options_list/suggestions_sorting'; export const getDefaultComponentState = (): OptionsListReduxState['componentState'] => ({ + popoverOpen: false, + allowExpensiveQueries: true, searchString: { value: '', valid: true }, }); @@ -41,6 +43,15 @@ export const optionsListReducers = { state.componentState.searchString.valid = getIpRangeQuery(action.payload).validSearch; } }, + setAllowExpensiveQueries: ( + state: WritableDraft, + action: PayloadAction + ) => { + state.componentState.allowExpensiveQueries = action.payload; + }, + setPopoverOpen: (state: WritableDraft, action: PayloadAction) => { + state.componentState.popoverOpen = action.payload; + }, setSort: ( state: WritableDraft, action: PayloadAction> @@ -97,7 +108,7 @@ export const optionsListReducers = { }, setField: ( state: WritableDraft, - action: PayloadAction + action: PayloadAction ) => { state.componentState.field = action.payload; }, @@ -110,7 +121,10 @@ export const optionsListReducers = { > > ) => { - state.componentState = { ...(state.componentState ?? {}), ...action.payload }; + state.componentState = { + ...(state.componentState ?? {}), + ...action.payload, + }; }, publishFilters: ( state: WritableDraft, diff --git a/src/plugins/controls/public/options_list/types.ts b/src/plugins/controls/public/options_list/types.ts index ae825132b6f80..c0501b0b6f38b 100644 --- a/src/plugins/controls/public/options_list/types.ts +++ b/src/plugins/controls/public/options_list/types.ts @@ -7,13 +7,17 @@ */ import { ReduxEmbeddableState } from '@kbn/presentation-util-plugin/public'; +import { FieldSpec } from '@kbn/data-views-plugin/common'; + import { ControlOutput } from '../types'; import { - OptionsListField, OptionsListSuggestions, OptionsListEmbeddableInput, } from '../../common/options_list/types'; +export const MIN_OPTIONS_LIST_REQUEST_SIZE = 10; +export const MAX_OPTIONS_LIST_REQUEST_SIZE = 1000; + interface SearchString { value: string; valid: boolean; @@ -21,12 +25,14 @@ interface SearchString { // Component state is only used by public components. export interface OptionsListComponentState { - field?: OptionsListField; - totalCardinality?: number; availableOptions?: OptionsListSuggestions; + allowExpensiveQueries: boolean; invalidSelections?: string[]; - validSelections?: string[]; searchString: SearchString; + validSelections?: string[]; + totalCardinality?: number; + popoverOpen: boolean; + field?: FieldSpec; } // public only - redux embeddable state type diff --git a/src/plugins/controls/public/services/http/http.stub.ts b/src/plugins/controls/public/services/http/http.stub.ts index d3362491ee71d..54fee10ce533e 100644 --- a/src/plugins/controls/public/services/http/http.stub.ts +++ b/src/plugins/controls/public/services/http/http.stub.ts @@ -13,5 +13,6 @@ import { ControlsHTTPService } from './types'; type HttpServiceFactory = PluginServiceFactory; export const httpServiceFactory: HttpServiceFactory = () => ({ + get: async () => ({} as unknown as HttpResponse), fetch: async () => ({} as unknown as HttpResponse), }); diff --git a/src/plugins/controls/public/services/http/http_service.ts b/src/plugins/controls/public/services/http/http_service.ts index 88eca23e3c1d6..c48aed61b1c12 100644 --- a/src/plugins/controls/public/services/http/http_service.ts +++ b/src/plugins/controls/public/services/http/http_service.ts @@ -16,10 +16,11 @@ export type HttpServiceFactory = KibanaPluginServiceFactory< >; export const httpServiceFactory: HttpServiceFactory = ({ coreStart }) => { const { - http: { fetch }, + http: { get, fetch }, } = coreStart; return { + get, fetch, }; }; diff --git a/src/plugins/controls/public/services/http/types.ts b/src/plugins/controls/public/services/http/types.ts index ce75c4b75a3d0..ea92ce6c4a74f 100644 --- a/src/plugins/controls/public/services/http/types.ts +++ b/src/plugins/controls/public/services/http/types.ts @@ -9,5 +9,6 @@ import { CoreSetup } from '@kbn/core/public'; export interface ControlsHTTPService { + get: CoreSetup['http']['get']; fetch: CoreSetup['http']['fetch']; } diff --git a/src/plugins/controls/public/services/options_list/options_list.story.ts b/src/plugins/controls/public/services/options_list/options_list.story.ts index c641e9ee8834a..6d3305f97b9aa 100644 --- a/src/plugins/controls/public/services/options_list/options_list.story.ts +++ b/src/plugins/controls/public/services/options_list/options_list.story.ts @@ -20,14 +20,11 @@ let optionsListRequestMethod = async (request: OptionsListRequest, abortSignal: r({ suggestions: {}, totalCardinality: 100, - rejected: false, }), 120 ) ); -const clearOptionsListCacheMock = () => {}; - export const replaceOptionsListMethod = ( newMethod: (request: OptionsListRequest, abortSignal: AbortSignal) => Promise ) => (optionsListRequestMethod = newMethod); @@ -35,6 +32,12 @@ export const replaceOptionsListMethod = ( export const optionsListServiceFactory: OptionsListServiceFactory = () => { return { runOptionsListRequest: optionsListRequestMethod, - clearOptionsListCache: clearOptionsListCacheMock, + clearOptionsListCache: jest.fn(), + getAllowExpensiveQueries: jest.fn().mockReturnValue(Promise.resolve(true)), + optionsListResponseWasFailure: jest + .fn() + .mockReturnValue( + false + ) as unknown as ControlsOptionsListService['optionsListResponseWasFailure'], }; }; diff --git a/src/plugins/controls/public/services/options_list/options_list_service.ts b/src/plugins/controls/public/services/options_list/options_list_service.ts index 888d2e2efb837..7152d190c997d 100644 --- a/src/plugins/controls/public/services/options_list/options_list_service.ts +++ b/src/plugins/controls/public/services/options_list/options_list_service.ts @@ -16,7 +16,7 @@ import { OptionsListRequest, OptionsListResponse, OptionsListRequestBody, - OptionsListField, + OptionsListFailureResponse, } from '../../../common/options_list/types'; import { ControlsHTTPService } from '../http/types'; import { ControlsDataService } from '../data/types'; @@ -38,6 +38,7 @@ class OptionsListService implements ControlsOptionsListService { private optionsListCacheResolver = (request: OptionsListRequest) => { const { + size, sort, query, filters, @@ -59,6 +60,7 @@ class OptionsListService implements ControlsOptionsListService { dataViewTitle, searchString, fieldName, + size, ].join('|'); }; @@ -90,18 +92,39 @@ class OptionsListService implements ControlsOptionsListService { filters: esFilters, fieldName: field.name, fieldSpec: field, - textFieldName: (field as OptionsListField).textFieldName, runtimeFieldMap: dataView.toSpec().runtimeFieldMap, }; }; + private cachedAllowExpensiveQueries = memoize(async () => { + const { allowExpensiveQueries } = await this.http.get<{ + allowExpensiveQueries: boolean; + }>('/api/kibana/controls/optionsList/getClusterSettings'); + return allowExpensiveQueries; + }); + + public getAllowExpensiveQueries = async (): Promise => { + try { + return await this.cachedAllowExpensiveQueries(); + } catch (error) { + return false; + } + }; + + public optionsListResponseWasFailure = ( + response: OptionsListResponse + ): response is OptionsListFailureResponse => { + return (response as OptionsListFailureResponse).error !== undefined; + }; + public runOptionsListRequest = async (request: OptionsListRequest, abortSignal: AbortSignal) => { try { return await this.cachedOptionsListRequest(request, abortSignal); - } catch (error) { + } catch (error: any) { // Remove rejected results from memoize cache this.cachedOptionsListRequest.cache.delete(this.optionsListCacheResolver(request)); - return { rejected: true } as OptionsListResponse; + if (error.name === 'AbortError') return { error: 'aborted' } as OptionsListFailureResponse; + return { error } as OptionsListFailureResponse; } }; diff --git a/src/plugins/controls/public/services/options_list/types.ts b/src/plugins/controls/public/services/options_list/types.ts index 569042b136419..c78819c27c5d5 100644 --- a/src/plugins/controls/public/services/options_list/types.ts +++ b/src/plugins/controls/public/services/options_list/types.ts @@ -6,13 +6,20 @@ * Side Public License, v 1. */ -import { OptionsListRequest, OptionsListResponse } from '../../../common/options_list/types'; +import { + OptionsListFailureResponse, + OptionsListRequest, + OptionsListResponse, +} from '../../../common/options_list/types'; export interface ControlsOptionsListService { runOptionsListRequest: ( request: OptionsListRequest, abortSignal: AbortSignal ) => Promise; - clearOptionsListCache: () => void; + optionsListResponseWasFailure: ( + response: OptionsListResponse + ) => response is OptionsListFailureResponse; + getAllowExpensiveQueries: () => Promise; } diff --git a/src/plugins/controls/public/types.ts b/src/plugins/controls/public/types.ts index 59686af51cca6..17608ee7bef8d 100644 --- a/src/plugins/controls/public/types.ts +++ b/src/plugins/controls/public/types.ts @@ -63,8 +63,6 @@ export interface ControlEditorProps { export interface DataControlField { field: DataViewField; - parentFieldName?: string; - childFieldName?: string; compatibleControlTypes: string[]; } diff --git a/src/plugins/controls/server/options_list/options_list_queries.test.ts b/src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.test.ts similarity index 57% rename from src/plugins/controls/server/options_list/options_list_queries.test.ts rename to src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.test.ts index d3e5525a36d06..31783a1267aca 100644 --- a/src/plugins/controls/server/options_list/options_list_queries.test.ts +++ b/src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.test.ts @@ -9,13 +9,10 @@ import { FieldSpec } from '@kbn/data-views-plugin/common'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; -import { - getSuggestionAggregationBuilder, - getValidationAggregationBuilder, -} from './options_list_queries'; +import { getCheapSuggestionAggregationBuilder } from './options_list_cheap_suggestion_queries'; import { OptionsListRequestBody } from '../../common/options_list/types'; -describe('options list queries', () => { +describe('options list cheap queries', () => { let rawSearchResponseMock: SearchResponse = {} as SearchResponse; beforeEach(() => { @@ -37,146 +34,85 @@ describe('options list queries', () => { }; }); - describe('validation aggregation and parsing', () => { - test('creates validation aggregation when given selections', () => { - const validationAggBuilder = getValidationAggregationBuilder(); - const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolTestField', - selectedOptions: ['coolOption1', 'coolOption2', 'coolOption3'], - }; - expect(validationAggBuilder.buildAggregation(optionsListRequestBodyMock)) - .toMatchInlineSnapshot(` - Object { - "filters": Object { - "filters": Object { - "coolOption1": Object { - "match": Object { - "coolTestField": "coolOption1", - }, - }, - "coolOption2": Object { - "match": Object { - "coolTestField": "coolOption2", - }, - }, - "coolOption3": Object { - "match": Object { - "coolTestField": "coolOption3", - }, - }, - }, - }, - } - `); - }); - - test('returns undefined when not given selections', () => { - const validationAggBuilder = getValidationAggregationBuilder(); - const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolTestField', - }; - expect(validationAggBuilder.buildAggregation(optionsListRequestBodyMock)).toBeUndefined(); - }); - - test('parses validation result', () => { - const validationAggBuilder = getValidationAggregationBuilder(); - rawSearchResponseMock.aggregations = { - validation: { - buckets: { - cool1: { doc_count: 0 }, - cool2: { doc_count: 15 }, - cool3: { doc_count: 0 }, - cool4: { doc_count: 2 }, - cool5: { doc_count: 112 }, - cool6: { doc_count: 0 }, - }, - }, - }; - expect(validationAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` - Array [ - "cool1", - "cool3", - "cool6", - ] - `); - }); - }); - describe('suggestion aggregation', () => { - describe('text / keyword field', () => { - test('with a search string, creates case insensitive aggregation', () => { + describe('keyword or text+keyword field', () => { + test('without a search string, creates keyword aggregation', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + allowExpensiveQueries: false, fieldName: 'coolTestField.keyword', - textFieldName: 'coolTestField', - searchString: 'cooool', + sort: { by: '_count', direction: 'asc' }, fieldSpec: { aggregatable: true } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "keywordSuggestions": Object { - "terms": Object { - "field": "coolTestField.keyword", - "order": Object { - "_count": "desc", - }, - "shard_size": 10, + "suggestions": Object { + "terms": Object { + "field": "coolTestField.keyword", + "include": ".*", + "order": Object { + "_count": "asc", }, - }, - }, - "filter": Object { - "match_phrase_prefix": Object { - "coolTestField": "cooool", + "shard_size": 10, }, }, } `); }); - test('without a search string, creates keyword aggregation', () => { + test('with a search string, creates case sensitive keyword aggregation', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + searchString: 'cooool', + allowExpensiveQueries: false, fieldName: 'coolTestField.keyword', - textFieldName: 'coolTestField', fieldSpec: { aggregatable: true } as unknown as FieldSpec, - sort: { by: '_count', direction: 'asc' }, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` - Object { - "terms": Object { - "execution_hint": "map", - "field": "coolTestField.keyword", - "include": ".*", - "order": Object { - "_count": "asc", + Object { + "suggestions": Object { + "terms": Object { + "field": "coolTestField.keyword", + "include": "cooool.*", + "order": Object { + "_count": "desc", + }, + "shard_size": 10, + }, }, - "shard_size": 10, - }, - } - `); + } + `); }); }); test('creates boolean aggregation for boolean field', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'coolean', - fieldSpec: { type: 'boolean' } as unknown as FieldSpec, + allowExpensiveQueries: false, sort: { by: '_key', direction: 'desc' }, + fieldSpec: { type: 'boolean' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "terms": Object { - "execution_hint": "map", - "field": "coolean", - "order": Object { - "_key": "desc", + "suggestions": Object { + "terms": Object { + "field": "coolean", + "order": Object { + "_key": "desc", + }, + "shard_size": 10, }, - "shard_size": 10, }, } `); @@ -184,53 +120,33 @@ describe('options list queries', () => { test('creates nested aggregation for nested field', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolNestedField', + size: 10, searchString: 'cooool', - fieldSpec: { subType: { nested: { path: 'path.to.nested' } } } as unknown as FieldSpec, + allowExpensiveQueries: false, + fieldName: 'coolNestedField', sort: { by: '_key', direction: 'asc' }, + fieldSpec: { subType: { nested: { path: 'path.to.nested' } } } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "nestedSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "coolNestedField", - "include": "cooool.*", - "order": Object { - "_key": "asc", + "nestedSuggestions": Object { + "aggs": Object { + "suggestions": Object { + "terms": Object { + "field": "coolNestedField", + "include": "cooool.*", + "order": Object { + "_key": "asc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "nested": Object { - "path": "path.to.nested", - }, - } - `); - }); - - test('creates keyword only aggregation', () => { - const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolTestField.keyword', - searchString: 'cooool', - fieldSpec: { aggregatable: true } as unknown as FieldSpec, - }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); - expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) - .toMatchInlineSnapshot(` - Object { - "terms": Object { - "execution_hint": "map", - "field": "coolTestField.keyword", - "include": "cooool.*", - "order": Object { - "_count": "desc", + "nested": Object { + "path": "path.to.nested", }, - "shard_size": 10, }, } `); @@ -239,36 +155,41 @@ describe('options list queries', () => { describe('IP field', () => { test('without a search string, creates IP range aggregation with default range', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', - fieldSpec: { type: 'ip' } as unknown as FieldSpec, + allowExpensiveQueries: false, sort: { by: '_count', direction: 'asc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "filteredSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "clientip", - "order": Object { - "_count": "asc", + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "asc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "ip_range": Object { - "field": "clientip", - "keyed": true, - "ranges": Array [ - Object { - "from": "::", - "key": "ipv6", - "to": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - }, - ], + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "::", + "key": "ipv6", + "to": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + }, + ], + }, }, } `); @@ -276,36 +197,41 @@ describe('options list queries', () => { test('full IPv4 in the search string, creates IP range aggregation with CIDR mask', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', - fieldSpec: { type: 'ip' } as unknown as FieldSpec, + allowExpensiveQueries: false, searchString: '41.77.243.255', sort: { by: '_key', direction: 'desc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "filteredSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "clientip", - "order": Object { - "_key": "desc", + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_key": "desc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "ip_range": Object { - "field": "clientip", - "keyed": true, - "ranges": Array [ - Object { - "key": "ipv4", - "mask": "41.77.243.255/32", - }, - ], + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "key": "ipv4", + "mask": "41.77.243.255/32", + }, + ], + }, }, } `); @@ -313,36 +239,41 @@ describe('options list queries', () => { test('full IPv6 in the search string, creates IP range aggregation with CIDR mask', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', + allowExpensiveQueries: false, + sort: { by: '_key', direction: 'asc' }, fieldSpec: { type: 'ip' } as unknown as FieldSpec, searchString: 'f688:fb50:6433:bba2:604:f2c:194a:d3c5', - sort: { by: '_key', direction: 'asc' }, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "filteredSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "clientip", - "order": Object { - "_key": "asc", + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_key": "asc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "ip_range": Object { - "field": "clientip", - "keyed": true, - "ranges": Array [ - Object { - "key": "ipv6", - "mask": "f688:fb50:6433:bba2:604:f2c:194a:d3c5/128", - }, - ], + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "key": "ipv6", + "mask": "f688:fb50:6433:bba2:604:f2c:194a:d3c5/128", + }, + ], + }, }, } `); @@ -350,36 +281,41 @@ describe('options list queries', () => { test('partial IPv4 in the search string, creates IP range aggregation with min and max', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', - fieldSpec: { type: 'ip' } as unknown as FieldSpec, searchString: '41.77', + allowExpensiveQueries: false, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "filteredSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "clientip", - "order": Object { - "_count": "desc", + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "desc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "ip_range": Object { - "field": "clientip", - "keyed": true, - "ranges": Array [ - Object { - "from": "41.77.0.0", - "key": "ipv4", - "to": "41.77.255.255", - }, - ], + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "41.77.0.0", + "key": "ipv4", + "to": "41.77.255.255", + }, + ], + }, }, } `); @@ -387,37 +323,42 @@ describe('options list queries', () => { test('partial IPv46 in the search string, creates IP range aggregation with min and max', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', - fieldSpec: { type: 'ip' } as unknown as FieldSpec, searchString: 'cdb6:', + allowExpensiveQueries: false, sort: { by: '_count', direction: 'desc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) .toMatchInlineSnapshot(` Object { - "aggs": Object { - "filteredSuggestions": Object { - "terms": Object { - "execution_hint": "map", - "field": "clientip", - "order": Object { - "_count": "desc", + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "desc", + }, + "shard_size": 10, }, - "shard_size": 10, }, }, - }, - "ip_range": Object { - "field": "clientip", - "keyed": true, - "ranges": Array [ - Object { - "from": "cdb6::", - "key": "ipv6", - "to": "cdb6:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - }, - ], + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "cdb6::", + "key": "ipv6", + "to": "cdb6:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + }, + ], + }, }, } `); @@ -428,24 +369,25 @@ describe('options list queries', () => { describe('suggestion parsing', () => { test('parses keyword / text result', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolTestField.keyword', - textFieldName: 'coolTestField', + size: 10, searchString: 'cooool', + allowExpensiveQueries: false, + fieldName: 'coolTestField.keyword', fieldSpec: { aggregatable: true } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); rawSearchResponseMock.aggregations = { suggestions: { - keywordSuggestions: { - buckets: [ - { doc_count: 5, key: 'cool1' }, - { doc_count: 15, key: 'cool2' }, - { doc_count: 10, key: 'cool3' }, - ], - }, + buckets: [ + { doc_count: 5, key: 'cool1' }, + { doc_count: 15, key: 'cool2' }, + { doc_count: 10, key: 'cool3' }, + ], }, }; - expect(suggestionAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` + expect( + suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock).suggestions + ).toMatchInlineSnapshot(` Object { "cool1": Object { "doc_count": 5, @@ -462,10 +404,12 @@ describe('options list queries', () => { test('parses boolean result', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'coolean', + allowExpensiveQueries: false, fieldSpec: { type: 'boolean' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); rawSearchResponseMock.aggregations = { suggestions: { buckets: [ @@ -474,7 +418,9 @@ describe('options list queries', () => { ], }, }; - expect(suggestionAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` + expect( + suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock).suggestions + ).toMatchInlineSnapshot(` Object { "false": Object { "doc_count": 55, @@ -488,14 +434,16 @@ describe('options list queries', () => { test('parses nested result', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolNestedField', + size: 10, searchString: 'cooool', + fieldName: 'coolNestedField', + allowExpensiveQueries: false, fieldSpec: { subType: { nested: { path: 'path.to.nested' } } } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); rawSearchResponseMock.aggregations = { - suggestions: { - nestedSuggestions: { + nestedSuggestions: { + suggestions: { buckets: [ { doc_count: 5, key: 'cool1' }, { doc_count: 15, key: 'cool2' }, @@ -504,7 +452,9 @@ describe('options list queries', () => { }, }, }; - expect(suggestionAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` + expect( + suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock).suggestions + ).toMatchInlineSnapshot(` Object { "cool1": Object { "doc_count": 5, @@ -521,11 +471,13 @@ describe('options list queries', () => { test('parses keyword only result', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { - fieldName: 'coolTestField.keyword', + size: 10, searchString: 'cooool', + allowExpensiveQueries: false, + fieldName: 'coolTestField.keyword', fieldSpec: { aggregatable: true } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); rawSearchResponseMock.aggregations = { suggestions: { buckets: [ @@ -535,7 +487,9 @@ describe('options list queries', () => { ], }, }; - expect(suggestionAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` + expect( + suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock).suggestions + ).toMatchInlineSnapshot(` Object { "cool1": Object { "doc_count": 5, @@ -553,10 +507,12 @@ describe('options list queries', () => { test('parses mixed IPv4 and IPv6 result', () => { const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, fieldName: 'clientip', + allowExpensiveQueries: false, fieldSpec: { type: 'ip' } as unknown as FieldSpec, }; - const suggestionAggBuilder = getSuggestionAggregationBuilder(optionsListRequestBodyMock); + const suggestionAggBuilder = getCheapSuggestionAggregationBuilder(optionsListRequestBodyMock); rawSearchResponseMock.aggregations = { suggestions: { buckets: { @@ -592,7 +548,10 @@ describe('options list queries', () => { }, }; - const parsed = suggestionAggBuilder.parse(rawSearchResponseMock); + const parsed = suggestionAggBuilder.parse( + rawSearchResponseMock, + optionsListRequestBodyMock + ).suggestions; /** first, verify that the sorting worked as expected */ expect(Object.keys(parsed)).toMatchInlineSnapshot(` Array [ diff --git a/src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.ts b/src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.ts new file mode 100644 index 0000000000000..3a302cf62d04b --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_cheap_suggestion_queries.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { get } from 'lodash'; +import { getFieldSubtypeNested } from '@kbn/data-views-plugin/common'; + +import { OptionsListRequestBody, OptionsListSuggestions } from '../../common/options_list/types'; +import { getIpRangeQuery, type IpRangeQuery } from '../../common/options_list/ip_search'; +import { EsBucket, OptionsListSuggestionAggregationBuilder } from './types'; +import { + getEscapedQuery, + getIpBuckets, + getSortType, +} from './options_list_suggestion_query_helpers'; + +/** + * Suggestion aggregations + */ +export const getCheapSuggestionAggregationBuilder = ({ fieldSpec }: OptionsListRequestBody) => { + if (fieldSpec?.type === 'boolean') { + return cheapSuggestionAggSubtypes.boolean; + } + if (fieldSpec?.type === 'ip') { + return cheapSuggestionAggSubtypes.ip; + } + if (fieldSpec && getFieldSubtypeNested(fieldSpec)) { + return cheapSuggestionAggSubtypes.subtypeNested; + } + return cheapSuggestionAggSubtypes.keywordOrText; +}; + +const cheapSuggestionAggSubtypes: { [key: string]: OptionsListSuggestionAggregationBuilder } = { + /** + * The "textOrKeyword" query / parser should be used whenever the field is built on some type non-nested string field + * (such as a keyword field or a keyword+text multi-field) + */ + keywordOrText: { + buildAggregation: ({ fieldName, searchString, sort }: OptionsListRequestBody) => ({ + suggestions: { + terms: { + field: fieldName, + include: `${getEscapedQuery(searchString)}.*`, + shard_size: 10, + order: getSortType(sort), + }, + }, + }), + parse: (rawEsResult) => ({ + suggestions: get(rawEsResult, 'aggregations.suggestions.buckets').reduce( + (suggestions: OptionsListSuggestions, suggestion: EsBucket) => { + return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; + }, + {} + ), + }), + }, + + /** + * the "Boolean" query / parser should be used when the options list is built on a field of type boolean. The query is slightly different than a keyword query. + */ + boolean: { + buildAggregation: ({ fieldName, sort }: OptionsListRequestBody) => ({ + suggestions: { + terms: { + field: fieldName, + shard_size: 10, + order: getSortType(sort), + }, + }, + }), + parse: (rawEsResult) => ({ + suggestions: get(rawEsResult, 'aggregations.suggestions.buckets')?.reduce( + (suggestions: OptionsListSuggestions, suggestion: EsBucket & { key_as_string: string }) => { + return { + ...suggestions, + [suggestion.key_as_string]: { doc_count: suggestion.doc_count }, + }; + }, + {} + ), + }), + }, + + /** + * the "IP" query / parser should be used when the options list is built on a field of type IP. + */ + ip: { + buildAggregation: ({ fieldName, searchString, sort }: OptionsListRequestBody) => { + let ipRangeQuery: IpRangeQuery = { + validSearch: true, + rangeQuery: [ + { + key: 'ipv6', + from: '::', + to: 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + }, + ], + }; + + if (searchString) { + ipRangeQuery = getIpRangeQuery(searchString); + if (!ipRangeQuery.validSearch) { + // ideally should be prevented on the client side but, if somehow an invalid search gets through to the server, + // simply don't return an aggregation query for the ES search request + return undefined; + } + } + + return { + suggestions: { + ip_range: { + field: fieldName, + ranges: ipRangeQuery.rangeQuery, + keyed: true, + }, + aggs: { + filteredSuggestions: { + terms: { + field: fieldName, + shard_size: 10, + order: getSortType(sort), + }, + }, + }, + }, + }; + }, + parse: (rawEsResult, { sort }) => { + if (!Boolean(rawEsResult.aggregations?.suggestions)) { + // if this is happens, that means there is an invalid search that snuck through to the server side code; + // so, might as well early return with no suggestions + return { suggestions: {} }; + } + + const buckets: EsBucket[] = []; + getIpBuckets(rawEsResult, buckets, 'ipv4'); // modifies buckets array directly, i.e. "by reference" + getIpBuckets(rawEsResult, buckets, 'ipv6'); + + const sortedSuggestions = + sort?.direction === 'asc' + ? buckets.sort( + (bucketA: EsBucket, bucketB: EsBucket) => bucketA.doc_count - bucketB.doc_count + ) + : buckets.sort( + (bucketA: EsBucket, bucketB: EsBucket) => bucketB.doc_count - bucketA.doc_count + ); + + return { + suggestions: sortedSuggestions + .slice(0, 10) // only return top 10 results + .reduce((suggestions, suggestion: EsBucket) => { + return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; + }, {}), + }; + }, + }, + + /** + * the "Subtype Nested" query / parser should be used when the options list is built on a field with subtype nested. + */ + subtypeNested: { + buildAggregation: (req: OptionsListRequestBody) => { + const { fieldSpec, fieldName, searchString, sort } = req; + const subTypeNested = fieldSpec && getFieldSubtypeNested(fieldSpec); + if (!subTypeNested) { + // if this field is not subtype nested, fall back to keywordOnly + return cheapSuggestionAggSubtypes.keywordOnly.buildAggregation(req); + } + return { + nestedSuggestions: { + nested: { + path: subTypeNested.nested.path, + }, + aggs: { + suggestions: { + terms: { + field: fieldName, + include: `${getEscapedQuery(searchString)}.*`, + shard_size: 10, + order: getSortType(sort), + }, + }, + }, + }, + }; + }, + parse: (rawEsResult) => ({ + suggestions: get(rawEsResult, 'aggregations.nestedSuggestions.suggestions.buckets').reduce( + (suggestions: OptionsListSuggestions, suggestion: EsBucket) => { + return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; + }, + {} + ), + }), + }, +}; diff --git a/src/plugins/controls/server/options_list/options_list_cluster_settings_route.ts b/src/plugins/controls/server/options_list/options_list_cluster_settings_route.ts new file mode 100644 index 0000000000000..3da1585405305 --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_cluster_settings_route.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getKbnServerError, reportServerError } from '@kbn/kibana-utils-plugin/server'; +import { CoreSetup } from '@kbn/core/server'; + +export const setupOptionsListClusterSettingsRoute = ({ http }: CoreSetup) => { + const router = http.createRouter(); + router.get( + { + path: '/api/kibana/controls/optionsList/getClusterSettings', + validate: false, + }, + async (context, _, response) => { + try { + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const settings = await esClient.cluster.getSettings({ + include_defaults: true, + filter_path: '**.allow_expensive_queries', + }); + + // priority: transient -> persistent -> default + const allowExpensiveQueries: string = + settings.transient?.search?.allow_expensive_queries ?? + settings.persistent?.search?.allow_expensive_queries ?? + settings.defaults?.search?.allow_expensive_queries ?? + // by default, the allowExpensiveQueries cluster setting is undefined; so, we need to treat this the same + // as `true` since that's the way other applications (such as the dashboard listing page) handle this. + 'true'; + + return response.ok({ + body: { + allowExpensiveQueries: allowExpensiveQueries === 'true', + }, + }); + } catch (e) { + const kbnErr = getKbnServerError(e); + return reportServerError(response, kbnErr); + } + } + ); +}; diff --git a/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.test.ts b/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.test.ts new file mode 100644 index 0000000000000..7026359e10ee4 --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.test.ts @@ -0,0 +1,676 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FieldSpec } from '@kbn/data-views-plugin/common'; +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; + +import { getExpensiveSuggestionAggregationBuilder } from './options_list_expensive_suggestion_queries'; +import { OptionsListRequestBody } from '../../common/options_list/types'; + +describe('options list expensive queries', () => { + let rawSearchResponseMock: SearchResponse = {} as SearchResponse; + + beforeEach(() => { + rawSearchResponseMock = { + hits: { + total: 10, + max_score: 10, + hits: [], + }, + took: 10, + timed_out: false, + _shards: { + failed: 0, + successful: 1, + total: 1, + skipped: 0, + }, + aggregations: {}, + }; + }); + + describe('suggestion aggregation', () => { + describe('string (keyword, text+keyword, or nested) field', () => { + test('test keyword field, without a search string', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + allowExpensiveQueries: true, + fieldName: 'coolTestField.keyword', + sort: { by: '_key', direction: 'asc' }, + fieldSpec: { aggregatable: true } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "terms": Object { + "field": "coolTestField.keyword", + "order": Object { + "_key": "asc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "coolTestField.keyword", + }, + }, + } + `); + }); + + test('test keyword field, with a search string', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + searchString: 'cooool', + allowExpensiveQueries: true, + fieldName: 'coolTestField.keyword', + sort: { by: '_key', direction: 'desc' }, + fieldSpec: { aggregatable: true } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "filteredSuggestions": Object { + "aggs": Object { + "suggestions": Object { + "terms": Object { + "field": "coolTestField.keyword", + "order": Object { + "_key": "desc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "coolTestField.keyword", + }, + }, + }, + "filter": Object { + "prefix": Object { + "coolTestField.keyword": Object { + "case_insensitive": true, + "value": "cooool", + }, + }, + }, + }, + } + `); + }); + + test('test nested field, with a search string', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + searchString: 'cooool', + allowExpensiveQueries: true, + fieldName: 'coolNestedField', + sort: { by: '_count', direction: 'asc' }, + fieldSpec: { subType: { nested: { path: 'path.to.nested' } } } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "nestedSuggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "aggs": Object { + "suggestions": Object { + "terms": Object { + "field": "coolNestedField", + "order": Object { + "_count": "asc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "coolNestedField", + }, + }, + }, + "filter": Object { + "prefix": Object { + "coolNestedField": Object { + "case_insensitive": true, + "value": "cooool", + }, + }, + }, + }, + }, + "nested": Object { + "path": "path.to.nested", + }, + }, + } + `); + }); + }); + + test('boolean field', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'coolean', + allowExpensiveQueries: false, + sort: { by: '_key', direction: 'desc' }, + fieldSpec: { type: 'boolean' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "terms": Object { + "field": "coolean", + "order": Object { + "_key": "desc", + }, + "shard_size": 10, + }, + }, + } + `); + }); + + describe('IP field', () => { + test('without a search string, creates IP range aggregation with default range', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + allowExpensiveQueries: true, + sort: { by: '_key', direction: 'asc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_key": "asc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "clientip", + }, + }, + }, + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "::", + "key": "ipv6", + "to": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + }, + ], + }, + }, + } + `); + }); + + test('full IPv4 in the search string, creates IP range aggregation with CIDR mask', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + allowExpensiveQueries: true, + searchString: '41.77.243.255', + sort: { by: '_count', direction: 'asc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "asc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "clientip", + }, + }, + }, + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "key": "ipv4", + "mask": "41.77.243.255/32", + }, + ], + }, + }, + } + `); + }); + + test('full IPv6 in the search string, creates IP range aggregation with CIDR mask', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + allowExpensiveQueries: true, + sort: { by: '_key', direction: 'asc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + searchString: 'f688:fb50:6433:bba2:604:f2c:194a:d3c5', + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_key": "asc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "clientip", + }, + }, + }, + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "key": "ipv6", + "mask": "f688:fb50:6433:bba2:604:f2c:194a:d3c5/128", + }, + ], + }, + }, + } + `); + }); + + test('partial IPv4 in the search string, creates IP range aggregation with min and max', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + searchString: '41.77', + allowExpensiveQueries: true, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "desc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "clientip", + }, + }, + }, + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "41.77.0.0", + "key": "ipv4", + "to": "41.77.255.255", + }, + ], + }, + }, + } + `); + }); + + test('partial IPv46 in the search string, creates IP range aggregation with min and max', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + searchString: 'cdb6:', + allowExpensiveQueries: true, + sort: { by: '_count', direction: 'desc' }, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + expect(suggestionAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "aggs": Object { + "filteredSuggestions": Object { + "terms": Object { + "field": "clientip", + "order": Object { + "_count": "desc", + }, + "shard_size": 10, + "size": 10, + }, + }, + "unique_terms": Object { + "cardinality": Object { + "field": "clientip", + }, + }, + }, + "ip_range": Object { + "field": "clientip", + "keyed": true, + "ranges": Array [ + Object { + "from": "cdb6::", + "key": "ipv6", + "to": "cdb6:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + }, + ], + }, + }, + } + `); + }); + }); + }); + + describe('suggestion parsing', () => { + test('parses string (keyword, text+keyword, or nested) result', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + allowExpensiveQueries: true, + fieldName: 'coolTestField.keyword', + fieldSpec: { aggregatable: true } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + rawSearchResponseMock.aggregations = { + suggestions: { + buckets: [ + { doc_count: 5, key: 'cool1' }, + { doc_count: 15, key: 'cool2' }, + { doc_count: 10, key: 'cool3' }, + ], + }, + unique_terms: { + value: 3, + }, + }; + expect(suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "cool1": Object { + "doc_count": 5, + }, + "cool2": Object { + "doc_count": 15, + }, + "cool3": Object { + "doc_count": 10, + }, + }, + "totalCardinality": 3, + } + `); + }); + + test('parses boolean result', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'coolean', + allowExpensiveQueries: true, + fieldSpec: { type: 'boolean' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + rawSearchResponseMock.aggregations = { + suggestions: { + buckets: [ + { doc_count: 55, key_as_string: 'false' }, + { doc_count: 155, key_as_string: 'true' }, + ], + }, + }; + expect(suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "false": Object { + "doc_count": 55, + }, + "true": Object { + "doc_count": 155, + }, + }, + "totalCardinality": 2, + } + `); + }); + + test('parses nested result', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + searchString: 'co', + fieldName: 'coolNestedField', + allowExpensiveQueries: true, + fieldSpec: { subType: { nested: { path: 'path.to.nested' } } } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + rawSearchResponseMock.aggregations = { + nestedSuggestions: { + filteredSuggestions: { + suggestions: { + buckets: [ + { doc_count: 5, key: 'cool1' }, + { doc_count: 15, key: 'cool2' }, + { doc_count: 10, key: 'cool3' }, + ], + }, + unique_terms: { + value: 3, + }, + }, + }, + }; + expect(suggestionAggBuilder.parse(rawSearchResponseMock, optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "suggestions": Object { + "cool1": Object { + "doc_count": 5, + }, + "cool2": Object { + "doc_count": 15, + }, + "cool3": Object { + "doc_count": 10, + }, + }, + "totalCardinality": 3, + } + `); + }); + + test('parses mixed IPv4 and IPv6 result', () => { + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'clientip', + allowExpensiveQueries: true, + fieldSpec: { type: 'ip' } as unknown as FieldSpec, + }; + const suggestionAggBuilder = getExpensiveSuggestionAggregationBuilder( + optionsListRequestBodyMock + ); + rawSearchResponseMock.aggregations = { + suggestions: { + buckets: { + ipv4: { + from: '0.0.0.0', + to: '255.255.255.255', + filteredSuggestions: { + buckets: [ + { doc_count: 8, key: '21.35.91.62' }, + { doc_count: 8, key: '21.35.91.61' }, + { doc_count: 11, key: '111.52.174.2' }, + { doc_count: 1, key: '56.73.58.63' }, + { doc_count: 9, key: '23.216.241.120' }, + { doc_count: 10, key: '196.162.13.39' }, + { doc_count: 7, key: '203.88.33.151' }, + ], + }, + }, + ipv6: { + from: '::', + to: 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + filteredSuggestions: { + buckets: [ + { doc_count: 12, key: '52:ae76:5947:5e2a:551:fe6a:712a:c72' }, + { doc_count: 1, key: 'fd:4aa0:c27c:b04:997f:2de1:51b4:8418' }, + { doc_count: 9, key: '28c7:c9a4:42fd:16b0:4de5:e41e:28d9:9172' }, + { doc_count: 6, key: '1ec:aa98:b0a6:d07c:590:18a0:8a33:2eb8' }, + { doc_count: 10, key: 'f7a9:640b:b5a0:1219:8d75:ed94:3c3e:2e63' }, + ], + }, + }, + }, + }, + unique_terms: { + buckets: { + ipv4: { + value: 7, + }, + ipv6: { + value: 5, + }, + }, + }, + }; + + const parsed = suggestionAggBuilder.parse( + rawSearchResponseMock, + optionsListRequestBodyMock + ).suggestions; + /** first, verify that the sorting worked as expected */ + expect(Object.keys(parsed)).toMatchInlineSnapshot(` + Array [ + "52:ae76:5947:5e2a:551:fe6a:712a:c72", + "111.52.174.2", + "196.162.13.39", + "f7a9:640b:b5a0:1219:8d75:ed94:3c3e:2e63", + "23.216.241.120", + "28c7:c9a4:42fd:16b0:4de5:e41e:28d9:9172", + "21.35.91.62", + "21.35.91.61", + "203.88.33.151", + "1ec:aa98:b0a6:d07c:590:18a0:8a33:2eb8", + ] + `); + /** then, make sure the object is structured properly */ + expect(parsed).toMatchInlineSnapshot(` + Object { + "111.52.174.2": Object { + "doc_count": 11, + }, + "196.162.13.39": Object { + "doc_count": 10, + }, + "1ec:aa98:b0a6:d07c:590:18a0:8a33:2eb8": Object { + "doc_count": 6, + }, + "203.88.33.151": Object { + "doc_count": 7, + }, + "21.35.91.61": Object { + "doc_count": 8, + }, + "21.35.91.62": Object { + "doc_count": 8, + }, + "23.216.241.120": Object { + "doc_count": 9, + }, + "28c7:c9a4:42fd:16b0:4de5:e41e:28d9:9172": Object { + "doc_count": 9, + }, + "52:ae76:5947:5e2a:551:fe6a:712a:c72": Object { + "doc_count": 12, + }, + "f7a9:640b:b5a0:1219:8d75:ed94:3c3e:2e63": Object { + "doc_count": 10, + }, + } + `); + }); + }); +}); diff --git a/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.ts b/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.ts new file mode 100644 index 0000000000000..63347f8d436d3 --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_expensive_suggestion_queries.ts @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { get } from 'lodash'; +import { getFieldSubtypeNested } from '@kbn/data-views-plugin/common'; + +import { OptionsListRequestBody, OptionsListSuggestions } from '../../common/options_list/types'; +import { getIpRangeQuery, type IpRangeQuery } from '../../common/options_list/ip_search'; +import { EsBucket, OptionsListSuggestionAggregationBuilder } from './types'; +import { getIpBuckets, getSortType } from './options_list_suggestion_query_helpers'; + +/** + * Suggestion aggregations + */ +export const getExpensiveSuggestionAggregationBuilder = ({ fieldSpec }: OptionsListRequestBody) => { + if (fieldSpec?.type === 'boolean') { + return expensiveSuggestionAggSubtypes.boolean; + } + if (fieldSpec?.type === 'ip') { + return expensiveSuggestionAggSubtypes.ip; + } + return expensiveSuggestionAggSubtypes.textOrKeywordOrNested; +}; + +const expensiveSuggestionAggSubtypes: { [key: string]: OptionsListSuggestionAggregationBuilder } = { + /** + * The "textOrKeywordOrNested" query / parser should be used whenever the field is built on some type of string field, + * regardless of if it is keyword only, keyword+text, or some nested keyword/keyword+text field. + */ + textOrKeywordOrNested: { + buildAggregation: ({ + searchString, + fieldName, + fieldSpec, + sort, + size, + }: OptionsListRequestBody) => { + const subTypeNested = fieldSpec && getFieldSubtypeNested(fieldSpec); + let textOrKeywordQuery: any = { + suggestions: { + terms: { + size, + field: fieldName, + shard_size: 10, + order: getSortType(sort), + }, + }, + unique_terms: { + cardinality: { + field: fieldName, + }, + }, + }; + if (searchString) { + textOrKeywordQuery = { + filteredSuggestions: { + filter: { + prefix: { + [fieldName]: { + value: searchString, + case_insensitive: true, + }, + }, + }, + aggs: { ...textOrKeywordQuery }, + }, + }; + } + if (subTypeNested) { + textOrKeywordQuery = { + nestedSuggestions: { + nested: { + path: subTypeNested.nested.path, + }, + aggs: { + ...textOrKeywordQuery, + }, + }, + }; + } + return textOrKeywordQuery; + }, + parse: (rawEsResult, request) => { + let basePath = 'aggregations'; + const isNested = request.fieldSpec && getFieldSubtypeNested(request.fieldSpec); + basePath += isNested ? '.nestedSuggestions' : ''; + basePath += request.searchString ? '.filteredSuggestions' : ''; + + const suggestions = get(rawEsResult, `${basePath}.suggestions.buckets`)?.reduce( + (acc: OptionsListSuggestions, suggestion: EsBucket) => { + return { ...acc, [suggestion.key]: { doc_count: suggestion.doc_count } }; + }, + {} + ); + return { + suggestions, + totalCardinality: get(rawEsResult, `${basePath}.unique_terms.value`), + }; + }, + }, + + /** + * the "Boolean" query / parser should be used when the options list is built on a field of type boolean. The query is slightly different than a keyword query. + */ + boolean: { + buildAggregation: ({ fieldName, sort }: OptionsListRequestBody) => ({ + suggestions: { + terms: { + field: fieldName, + shard_size: 10, + order: getSortType(sort), + }, + }, + }), + parse: (rawEsResult) => { + const suggestions = get(rawEsResult, 'aggregations.suggestions.buckets')?.reduce( + (acc: OptionsListSuggestions, suggestion: EsBucket & { key_as_string: string }) => { + return { + ...acc, + [suggestion.key_as_string]: { doc_count: suggestion.doc_count }, + }; + }, + {} + ); + return { suggestions, totalCardinality: Object.keys(suggestions).length }; // cardinality is only ever 0, 1, or 2 so safe to use length here + }, + }, + + /** + * the "IP" query / parser should be used when the options list is built on a field of type IP. + */ + ip: { + buildAggregation: ({ fieldName, searchString, sort, size }: OptionsListRequestBody) => { + let ipRangeQuery: IpRangeQuery = { + validSearch: true, + rangeQuery: [ + { + key: 'ipv6', + from: '::', + to: 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + }, + ], + }; + + if (searchString) { + ipRangeQuery = getIpRangeQuery(searchString); + if (!ipRangeQuery.validSearch) { + // ideally should be prevented on the client side but, if somehow an invalid search gets through to the server, + // simply don't return an aggregation query for the ES search request + return undefined; + } + } + + return { + suggestions: { + ip_range: { + field: fieldName, + ranges: ipRangeQuery.rangeQuery, + keyed: true, + }, + aggs: { + filteredSuggestions: { + terms: { + size, + field: fieldName, + shard_size: 10, + order: getSortType(sort), + }, + }, + unique_terms: { + cardinality: { + field: fieldName, + }, + }, + }, + }, + }; + }, + parse: (rawEsResult, request) => { + if (!Boolean(rawEsResult.aggregations?.suggestions)) { + // if this is happens, that means there is an invalid search that snuck through to the server side code; + // so, might as well early return with no suggestions + return { suggestions: {}, totalCardinality: 0 }; + } + const buckets: EsBucket[] = []; + getIpBuckets(rawEsResult, buckets, 'ipv4'); // modifies buckets array directly, i.e. "by reference" + getIpBuckets(rawEsResult, buckets, 'ipv6'); + + const sortedSuggestions = + request.sort?.direction === 'asc' + ? buckets.sort( + (bucketA: EsBucket, bucketB: EsBucket) => bucketA.doc_count - bucketB.doc_count + ) + : buckets.sort( + (bucketA: EsBucket, bucketB: EsBucket) => bucketB.doc_count - bucketA.doc_count + ); + + const suggestions: OptionsListSuggestions = sortedSuggestions + .slice(0, request.size) + .reduce((acc: OptionsListSuggestions, suggestion: EsBucket) => { + return { ...acc, [suggestion.key]: { doc_count: suggestion.doc_count } }; + }, {}); + const totalCardinality = + (get(rawEsResult, `aggregations.suggestions.buckets.ipv4.unique_terms.value`) ?? 0) + + (get(rawEsResult, `aggregations.suggestions.buckets.ipv6.unique_terms.value`) ?? 0); + return { + suggestions, + totalCardinality, + }; + }, + }, +}; diff --git a/src/plugins/controls/server/options_list/options_list_queries.ts b/src/plugins/controls/server/options_list/options_list_queries.ts deleted file mode 100644 index 4a7aceba910c0..0000000000000 --- a/src/plugins/controls/server/options_list/options_list_queries.ts +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { get, isEmpty } from 'lodash'; -import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; -import { getFieldSubtypeNested } from '@kbn/data-views-plugin/common'; - -import { OptionsListRequestBody, OptionsListSuggestions } from '../../common/options_list/types'; -import { - OPTIONS_LIST_DEFAULT_SORT, - OptionsListSortingType, -} from '../../common/options_list/suggestions_sorting'; -import { getIpRangeQuery, type IpRangeQuery } from '../../common/options_list/ip_search'; - -export interface OptionsListValidationAggregationBuilder { - buildAggregation: (req: OptionsListRequestBody) => unknown; - parse: (response: SearchResponse) => string[]; -} - -export interface OptionsListSuggestionAggregationBuilder { - buildAggregation: (req: OptionsListRequestBody) => unknown; - parse: (response: SearchResponse) => OptionsListSuggestions; -} - -interface EsBucket { - key: string; - doc_count: number; -} - -const getSortType = (sort?: OptionsListSortingType) => { - return sort - ? { [sort.by]: sort.direction } - : { [OPTIONS_LIST_DEFAULT_SORT.by]: OPTIONS_LIST_DEFAULT_SORT.direction }; -}; - -/** - * Validation aggregations - */ -export const getValidationAggregationBuilder: () => OptionsListValidationAggregationBuilder = - () => ({ - buildAggregation: ({ selectedOptions, fieldName }: OptionsListRequestBody) => { - let selectedOptionsFilters; - if (selectedOptions) { - selectedOptionsFilters = selectedOptions.reduce((acc, currentOption) => { - acc[currentOption] = { match: { [fieldName]: currentOption } }; - return acc; - }, {} as { [key: string]: { match: { [key: string]: string } } }); - } - return selectedOptionsFilters && !isEmpty(selectedOptionsFilters) - ? { - filters: { - filters: selectedOptionsFilters, - }, - } - : undefined; - }, - parse: (rawEsResult) => { - const rawInvalidSuggestions = get(rawEsResult, 'aggregations.validation.buckets'); - return rawInvalidSuggestions && !isEmpty(rawInvalidSuggestions) - ? Object.keys(rawInvalidSuggestions).filter( - (key) => rawInvalidSuggestions[key].doc_count === 0 - ) - : []; - }, - }); - -/** - * Suggestion aggregations - */ -export const getSuggestionAggregationBuilder = ({ - fieldSpec, - textFieldName, - searchString, -}: OptionsListRequestBody) => { - if (textFieldName && fieldSpec?.aggregatable && searchString) { - return suggestionAggSubtypes.keywordAndText; - } - if (fieldSpec?.type === 'boolean') { - return suggestionAggSubtypes.boolean; - } - if (fieldSpec?.type === 'ip') { - return suggestionAggSubtypes.ip; - } - if (fieldSpec && getFieldSubtypeNested(fieldSpec)) { - return suggestionAggSubtypes.subtypeNested; - } - return suggestionAggSubtypes.keywordOnly; -}; - -const getEscapedQuery = (q: string = '') => - q.replace(/[.?+*|{}[\]()"\\#@&<>~]/g, (match) => `\\${match}`); - -const getIpBuckets = (rawEsResult: any, combinedBuckets: EsBucket[], type: 'ipv4' | 'ipv6') => { - const results = get( - rawEsResult, - `aggregations.suggestions.buckets.${type}.filteredSuggestions.buckets` - ); - if (results) { - results.forEach((suggestion: EsBucket) => combinedBuckets.push(suggestion)); - } -}; - -const suggestionAggSubtypes: { [key: string]: OptionsListSuggestionAggregationBuilder } = { - /** - * the "Keyword only" query / parser should be used when the options list is built on a field which has only keyword mappings. - */ - keywordOnly: { - buildAggregation: ({ fieldName, searchString, sort }: OptionsListRequestBody) => ({ - terms: { - field: fieldName, - include: `${getEscapedQuery(searchString)}.*`, - execution_hint: 'map', - shard_size: 10, - order: getSortType(sort), - }, - }), - parse: (rawEsResult) => - get(rawEsResult, 'aggregations.suggestions.buckets').reduce( - (suggestions: OptionsListSuggestions, suggestion: EsBucket) => { - return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; - }, - {} - ), - }, - - /** - * the "Keyword and text" query / parser should be used when the options list is built on a multi-field which has both keyword and text mappings. It supports case-insensitive searching - */ - keywordAndText: { - buildAggregation: (req: OptionsListRequestBody) => { - if (!req.textFieldName) { - // if there is no textFieldName specified, or if there is no search string yet fall back to keywordOnly - return suggestionAggSubtypes.keywordOnly.buildAggregation(req); - } - const { fieldName, searchString, textFieldName, sort } = req; - return { - filter: { - match_phrase_prefix: { - [textFieldName]: searchString, - }, - }, - aggs: { - keywordSuggestions: { - terms: { - field: fieldName, - shard_size: 10, - order: getSortType(sort), - }, - }, - }, - }; - }, - parse: (rawEsResult) => - get(rawEsResult, 'aggregations.suggestions.keywordSuggestions.buckets').reduce( - (suggestions: OptionsListSuggestions, suggestion: EsBucket) => { - return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; - }, - {} - ), - }, - - /** - * the "Boolean" query / parser should be used when the options list is built on a field of type boolean. The query is slightly different than a keyword query. - */ - boolean: { - buildAggregation: ({ fieldName, sort }: OptionsListRequestBody) => ({ - terms: { - field: fieldName, - execution_hint: 'map', - shard_size: 10, - order: getSortType(sort), - }, - }), - parse: (rawEsResult) => - get(rawEsResult, 'aggregations.suggestions.buckets')?.reduce( - (suggestions: OptionsListSuggestions, suggestion: EsBucket & { key_as_string: string }) => { - return { - ...suggestions, - [suggestion.key_as_string]: { doc_count: suggestion.doc_count }, - }; - }, - {} - ), - }, - - /** - * the "IP" query / parser should be used when the options list is built on a field of type IP. - */ - ip: { - buildAggregation: ({ fieldName, searchString, sort }: OptionsListRequestBody) => { - let ipRangeQuery: IpRangeQuery = { - validSearch: true, - rangeQuery: [ - { - key: 'ipv6', - from: '::', - to: 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', - }, - ], - }; - - if (searchString) { - ipRangeQuery = getIpRangeQuery(searchString); - if (!ipRangeQuery.validSearch) { - // ideally should be prevented on the client side but, if somehow an invalid search gets through to the server, - // simply don't return an aggregation query for the ES search request - return undefined; - } - } - - return { - ip_range: { - field: fieldName, - ranges: ipRangeQuery.rangeQuery, - keyed: true, - }, - aggs: { - filteredSuggestions: { - terms: { - field: fieldName, - execution_hint: 'map', - shard_size: 10, - order: getSortType(sort), - }, - }, - }, - }; - }, - parse: (rawEsResult) => { - if (!Boolean(rawEsResult.aggregations?.suggestions)) { - // if this is happens, that means there is an invalid search that snuck through to the server side code; - // so, might as well early return with no suggestions - return []; - } - - const buckets: EsBucket[] = []; - getIpBuckets(rawEsResult, buckets, 'ipv4'); // modifies buckets array directly, i.e. "by reference" - getIpBuckets(rawEsResult, buckets, 'ipv6'); - return buckets - .sort((bucketA: EsBucket, bucketB: EsBucket) => bucketB.doc_count - bucketA.doc_count) - .slice(0, 10) // only return top 10 results - .reduce((suggestions, suggestion: EsBucket) => { - return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; - }, {}); - }, - }, - - /** - * the "Subtype Nested" query / parser should be used when the options list is built on a field with subtype nested. - */ - subtypeNested: { - buildAggregation: (req: OptionsListRequestBody) => { - const { fieldSpec, fieldName, searchString, sort } = req; - const subTypeNested = fieldSpec && getFieldSubtypeNested(fieldSpec); - if (!subTypeNested) { - // if this field is not subtype nested, fall back to keywordOnly - return suggestionAggSubtypes.keywordOnly.buildAggregation(req); - } - return { - nested: { - path: subTypeNested.nested.path, - }, - aggs: { - nestedSuggestions: { - terms: { - field: fieldName, - include: `${getEscapedQuery(searchString)}.*`, - execution_hint: 'map', - shard_size: 10, - order: getSortType(sort), - }, - }, - }, - }; - }, - parse: (rawEsResult) => - get(rawEsResult, 'aggregations.suggestions.nestedSuggestions.buckets').reduce( - (suggestions: OptionsListSuggestions, suggestion: EsBucket) => { - return { ...suggestions, [suggestion.key]: { doc_count: suggestion.doc_count } }; - }, - {} - ), - }, -}; diff --git a/src/plugins/controls/server/options_list/options_list_suggestion_query_helpers.ts b/src/plugins/controls/server/options_list/options_list_suggestion_query_helpers.ts new file mode 100644 index 0000000000000..54e83e0ac835b --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_suggestion_query_helpers.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { get } from 'lodash'; + +import { EsBucket } from './types'; +import { + OPTIONS_LIST_DEFAULT_SORT, + OptionsListSortingType, +} from '../../common/options_list/suggestions_sorting'; + +export const getSortType = (sort?: OptionsListSortingType) => { + return sort + ? { [sort.by]: sort.direction } + : { [OPTIONS_LIST_DEFAULT_SORT.by]: OPTIONS_LIST_DEFAULT_SORT.direction }; +}; + +export const getEscapedQuery = (q: string = '') => + q.replace(/[.?+*|{}[\]()"\\#@&<>~]/g, (match) => `\\${match}`); + +export const getIpBuckets = ( + rawEsResult: any, + combinedBuckets: EsBucket[], + type: 'ipv4' | 'ipv6' +) => { + const results = get( + rawEsResult, + `aggregations.suggestions.buckets.${type}.filteredSuggestions.buckets` + ); + if (results) { + results.forEach((suggestion: EsBucket) => combinedBuckets.push(suggestion)); + } +}; diff --git a/src/plugins/controls/server/options_list/options_list_suggestions_route.ts b/src/plugins/controls/server/options_list/options_list_suggestions_route.ts index 0893d24ebacf0..61d50419d94a0 100644 --- a/src/plugins/controls/server/options_list/options_list_suggestions_route.ts +++ b/src/plugins/controls/server/options_list/options_list_suggestions_route.ts @@ -7,7 +7,6 @@ */ import { Observable } from 'rxjs'; -import { get } from 'lodash'; import { PluginSetup as UnifiedSearchPluginSetup } from '@kbn/unified-search-plugin/server'; import { getKbnServerError, reportServerError } from '@kbn/kibana-utils-plugin/server'; @@ -16,10 +15,10 @@ import { SearchRequest } from '@kbn/data-plugin/common'; import { schema } from '@kbn/config-schema'; import { OptionsListRequestBody, OptionsListResponse } from '../../common/options_list/types'; -import { - getSuggestionAggregationBuilder, - getValidationAggregationBuilder, -} from './options_list_queries'; +import { getValidationAggregationBuilder } from './options_list_validation_queries'; +import { getExpensiveSuggestionAggregationBuilder } from './options_list_expensive_suggestion_queries'; +import { getCheapSuggestionAggregationBuilder } from './options_list_cheap_suggestion_queries'; +import { OptionsListSuggestionAggregationBuilder } from './types'; export const setupOptionsListSuggestionsRoute = ( { http }: CoreSetup, @@ -39,10 +38,12 @@ export const setupOptionsListSuggestionsRoute = ( ), body: schema.object( { + size: schema.number(), fieldName: schema.string(), sort: schema.maybe(schema.any()), filters: schema.maybe(schema.any()), fieldSpec: schema.maybe(schema.any()), + allowExpensiveQueries: schema.boolean(), searchString: schema.maybe(schema.string()), selectedOptions: schema.maybe(schema.arrayOf(schema.string())), }, @@ -55,6 +56,7 @@ export const setupOptionsListSuggestionsRoute = ( const suggestionRequest: OptionsListRequestBody = request.body; const { index } = request.params; const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const suggestionsResponse = await getOptionsListSuggestions({ abortedEvent$: request.events.aborted$, request: suggestionRequest, @@ -86,21 +88,21 @@ export const setupOptionsListSuggestionsRoute = ( /** * Build ES Query */ - const { runPastTimeout, filters, fieldName, runtimeFieldMap } = request; + const { runPastTimeout, filters, runtimeFieldMap, allowExpensiveQueries } = request; const { terminateAfter, timeout } = getAutocompleteSettings(); const timeoutSettings = runPastTimeout ? {} : { timeout: `${timeout}ms`, terminate_after: terminateAfter }; - const suggestionBuilder = getSuggestionAggregationBuilder(request); + let suggestionBuilder: OptionsListSuggestionAggregationBuilder; + if (allowExpensiveQueries) { + suggestionBuilder = getExpensiveSuggestionAggregationBuilder(request); + } else { + suggestionBuilder = getCheapSuggestionAggregationBuilder(request); + } const validationBuilder = getValidationAggregationBuilder(); - const builtSuggestionAggregation = suggestionBuilder.buildAggregation(request); - const suggestionAggregation = builtSuggestionAggregation - ? { - suggestions: builtSuggestionAggregation, - } - : {}; + const suggestionAggregation: any = suggestionBuilder.buildAggregation(request) ?? {}; const builtValidationAggregation = validationBuilder.buildAggregation(request); const validationAggregations = builtValidationAggregation ? { @@ -118,17 +120,11 @@ export const setupOptionsListSuggestionsRoute = ( aggs: { ...suggestionAggregation, ...validationAggregations, - unique_terms: { - cardinality: { - field: fieldName, - }, - }, }, runtime_mappings: { ...runtimeFieldMap, }, }; - /** * Run ES query */ @@ -137,14 +133,13 @@ export const setupOptionsListSuggestionsRoute = ( /** * Parse ES response into Options List Response */ - const totalCardinality = get(rawEsResult, 'aggregations.unique_terms.value'); - const suggestions = suggestionBuilder.parse(rawEsResult); + const results = suggestionBuilder.parse(rawEsResult, request); + const totalCardinality = results.totalCardinality; const invalidSelections = validationBuilder.parse(rawEsResult); return { - suggestions, + suggestions: results.suggestions, totalCardinality, invalidSelections, - rejected: false, }; }; }; diff --git a/src/plugins/controls/server/options_list/options_list_validation_queries.test.ts b/src/plugins/controls/server/options_list/options_list_validation_queries.test.ts new file mode 100644 index 0000000000000..f8a7344eb8860 --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_validation_queries.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; + +import { OptionsListRequestBody } from '../../common/options_list/types'; +import { getValidationAggregationBuilder } from './options_list_validation_queries'; + +describe('options list queries', () => { + let rawSearchResponseMock: SearchResponse = {} as SearchResponse; + + beforeEach(() => { + rawSearchResponseMock = { + hits: { + total: 10, + max_score: 10, + hits: [], + }, + took: 10, + timed_out: false, + _shards: { + failed: 0, + successful: 1, + total: 1, + skipped: 0, + }, + aggregations: {}, + }; + }); + + describe('validation aggregation and parsing', () => { + test('creates validation aggregation when given selections', () => { + const validationAggBuilder = getValidationAggregationBuilder(); + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'coolTestField', + allowExpensiveQueries: true, + selectedOptions: ['coolOption1', 'coolOption2', 'coolOption3'], + }; + expect(validationAggBuilder.buildAggregation(optionsListRequestBodyMock)) + .toMatchInlineSnapshot(` + Object { + "filters": Object { + "filters": Object { + "coolOption1": Object { + "match": Object { + "coolTestField": "coolOption1", + }, + }, + "coolOption2": Object { + "match": Object { + "coolTestField": "coolOption2", + }, + }, + "coolOption3": Object { + "match": Object { + "coolTestField": "coolOption3", + }, + }, + }, + }, + } + `); + }); + + test('returns undefined when not given selections', () => { + const validationAggBuilder = getValidationAggregationBuilder(); + const optionsListRequestBodyMock: OptionsListRequestBody = { + size: 10, + fieldName: 'coolTestField', + allowExpensiveQueries: true, + }; + expect(validationAggBuilder.buildAggregation(optionsListRequestBodyMock)).toBeUndefined(); + }); + + test('parses validation result', () => { + const validationAggBuilder = getValidationAggregationBuilder(); + rawSearchResponseMock.aggregations = { + validation: { + buckets: { + cool1: { doc_count: 0 }, + cool2: { doc_count: 15 }, + cool3: { doc_count: 0 }, + cool4: { doc_count: 2 }, + cool5: { doc_count: 112 }, + cool6: { doc_count: 0 }, + }, + }, + }; + expect(validationAggBuilder.parse(rawSearchResponseMock)).toMatchInlineSnapshot(` + Array [ + "cool1", + "cool3", + "cool6", + ] + `); + }); + }); +}); diff --git a/src/plugins/controls/server/options_list/options_list_validation_queries.ts b/src/plugins/controls/server/options_list/options_list_validation_queries.ts new file mode 100644 index 0000000000000..a79c0971a5cb1 --- /dev/null +++ b/src/plugins/controls/server/options_list/options_list_validation_queries.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { get, isEmpty } from 'lodash'; + +import { OptionsListRequestBody } from '../../common/options_list/types'; +import { OptionsListValidationAggregationBuilder } from './types'; + +/** + * Validation aggregations + */ +export const getValidationAggregationBuilder: () => OptionsListValidationAggregationBuilder = + () => ({ + buildAggregation: ({ selectedOptions, fieldName }: OptionsListRequestBody) => { + let selectedOptionsFilters; + if (selectedOptions) { + selectedOptionsFilters = selectedOptions.reduce((acc, currentOption) => { + acc[currentOption] = { match: { [fieldName]: currentOption } }; + return acc; + }, {} as { [key: string]: { match: { [key: string]: string } } }); + } + return selectedOptionsFilters && !isEmpty(selectedOptionsFilters) + ? { + filters: { + filters: selectedOptionsFilters, + }, + } + : undefined; + }, + parse: (rawEsResult) => { + const rawInvalidSuggestions = get(rawEsResult, 'aggregations.validation.buckets'); + return rawInvalidSuggestions && !isEmpty(rawInvalidSuggestions) + ? Object.keys(rawInvalidSuggestions).filter( + (key) => rawInvalidSuggestions[key].doc_count === 0 + ) + : []; + }, + }); diff --git a/src/plugins/controls/server/options_list/types.ts b/src/plugins/controls/server/options_list/types.ts new file mode 100644 index 0000000000000..1ea3475eddffb --- /dev/null +++ b/src/plugins/controls/server/options_list/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import { + OptionsListRequestBody, + OptionsListParsedSuggestions, +} from '../../common/options_list/types'; + +export interface EsBucket { + key: string; + doc_count: number; +} + +export interface OptionsListValidationAggregationBuilder { + buildAggregation: (req: OptionsListRequestBody) => unknown; + parse: (response: SearchResponse) => string[]; +} + +export interface OptionsListSuggestionAggregationBuilder { + buildAggregation: (req: OptionsListRequestBody) => unknown; + parse: (response: SearchResponse, req: OptionsListRequestBody) => OptionsListParsedSuggestions; +} diff --git a/src/plugins/controls/server/plugin.ts b/src/plugins/controls/server/plugin.ts index 6d3838804e14a..7e394ca69a410 100644 --- a/src/plugins/controls/server/plugin.ts +++ b/src/plugins/controls/server/plugin.ts @@ -15,6 +15,7 @@ import { controlGroupContainerPersistableStateServiceFactory } from './control_g import { optionsListPersistableStateServiceFactory } from './options_list/options_list_embeddable_factory'; import { rangeSliderPersistableStateServiceFactory } from './range_slider/range_slider_embeddable_factory'; import { timeSliderPersistableStateServiceFactory } from './time_slider/time_slider_embeddable_factory'; +import { setupOptionsListClusterSettingsRoute } from './options_list/options_list_cluster_settings_route'; interface SetupDeps { embeddable: EmbeddableSetup; @@ -30,7 +31,7 @@ export class ControlsPlugin implements Plugin { embeddable.registerEmbeddableFactory(optionsListPersistableStateServiceFactory()); embeddable.registerEmbeddableFactory(rangeSliderPersistableStateServiceFactory()); embeddable.registerEmbeddableFactory(timeSliderPersistableStateServiceFactory()); - + setupOptionsListClusterSettingsRoute(core); setupOptionsListSuggestionsRoute(core, unifiedSearch.autocomplete.getAutocompleteSettings); return {}; } diff --git a/src/plugins/controls/tsconfig.json b/src/plugins/controls/tsconfig.json index 6aedcaf236950..9acd44b5c2f7e 100644 --- a/src/plugins/controls/tsconfig.json +++ b/src/plugins/controls/tsconfig.json @@ -32,6 +32,7 @@ "@kbn/test-jest-helpers", "@kbn/config-schema", "@kbn/storybook", + "@kbn/ui-theme", ], "exclude": [ "target/**/*", diff --git a/src/plugins/dashboard/public/dashboard_container/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/dashboard_container/embeddable/dashboard_container.tsx index cdad643417538..c2205d68ba546 100644 --- a/src/plugins/dashboard/public/dashboard_container/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/dashboard_container/embeddable/dashboard_container.tsx @@ -96,7 +96,7 @@ export class DashboardContainer extends Container void; + public onDestroyControlGroup?: () => void; private subscriptions: Subscription = new Subscription(); private initialized$: BehaviorSubject = new BehaviorSubject(false); diff --git a/src/plugins/dashboard/public/dashboard_container/embeddable/integrations/controls/dashboard_control_group_integration.ts b/src/plugins/dashboard/public/dashboard_container/embeddable/integrations/controls/dashboard_control_group_integration.ts index 4e78d9185f7c4..7367f089bbaa8 100644 --- a/src/plugins/dashboard/public/dashboard_container/embeddable/integrations/controls/dashboard_control_group_integration.ts +++ b/src/plugins/dashboard/public/dashboard_container/embeddable/integrations/controls/dashboard_control_group_integration.ts @@ -65,12 +65,19 @@ export async function startControlGroupIntegration( return; } - this.untilInitialized().then(() => startSyncingDashboardControlGroup.bind(this)()); + this.untilInitialized().then(() => { + const stopSyncingControlGroup = + startSyncingDashboardControlGroup.bind(this)()?.stopSyncingWithControlGroup; + this.onDestroyControlGroup = () => { + stopSyncingControlGroup?.(); + this.controlGroup?.destroy(); + }; + }); await controlGroup.untilInitialized(); return controlGroup; } -async function startSyncingDashboardControlGroup(this: DashboardContainer) { +function startSyncingDashboardControlGroup(this: DashboardContainer) { if (!this.controlGroup) return; const subscriptions = new Subscription(); diff --git a/src/plugins/embeddable/public/lib/embeddables/error_embedabble.scss b/src/plugins/embeddable/public/lib/embeddables/error_embedabble.scss deleted file mode 100644 index 51ee8cdf25f66..0000000000000 --- a/src/plugins/embeddable/public/lib/embeddables/error_embedabble.scss +++ /dev/null @@ -1,10 +0,0 @@ -.errorEmbeddableCompact__popoverAnchor { - max-width: 100%; -} - -.errorEmbeddableCompact__button { - padding-left: 0 !important; - .euiIcon { - margin-right: $euiSizeXS * .5; - } -} \ No newline at end of file diff --git a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.scss b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.scss new file mode 100644 index 0000000000000..cc8d7f9698221 --- /dev/null +++ b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.scss @@ -0,0 +1,20 @@ + +.errorEmbeddableCompact__popover { + height: 100%; +} + +.errorEmbeddableCompact__popoverAnchor { + max-width: 100%; +} + +.errorEmbeddableCompact__button { + :hover { + text-decoration: underline; + text-decoration-color: $euiTextSubduedColor !important; + } +} + +.errorEmbeddableCompact__text { + font-size: $euiSizeM; + color: $euiTextSubduedColor; +} diff --git a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx index 1ccaf61064a8a..244803c5e13f2 100644 --- a/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/error_embeddable.tsx @@ -11,6 +11,7 @@ import { EmbeddablePanelError } from '../panel/embeddable_panel_error'; import { Embeddable } from './embeddable'; import { EmbeddableInput, EmbeddableOutput, IEmbeddable } from './i_embeddable'; import { IContainer } from '../containers'; +import './error_embeddable.scss'; export const ERROR_EMBEDDABLE_TYPE = 'error'; diff --git a/test/functional/apps/dashboard_elements/controls/options_list/index.ts b/test/functional/apps/dashboard_elements/controls/options_list/index.ts index 9d4cb7d18d525..d2c80da466c53 100644 --- a/test/functional/apps/dashboard_elements/controls/options_list/index.ts +++ b/test/functional/apps/dashboard_elements/controls/options_list/index.ts @@ -14,16 +14,12 @@ export default function ({ loadTestFile, getService, getPageObjects }: FtrProvid const elasticChart = getService('elasticChart'); const security = getService('security'); - const { timePicker, dashboard } = getPageObjects([ - 'dashboardControls', - 'timePicker', - 'dashboard', - 'common', - ]); - - async function setup() { + const { timePicker, dashboard, common } = getPageObjects(['timePicker', 'dashboard', 'common']); + + const setup = async () => { await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader', 'animals']); + await common.navigateToApp('dashboard'); await dashboard.gotoDashboardLandingPage(); await dashboard.clickNewDashboard(); await timePicker.setDefaultDataRange(); @@ -32,13 +28,13 @@ export default function ({ loadTestFile, getService, getPageObjects }: FtrProvid exitFromEditMode: false, storeTimeWithDashboard: true, }); - } + }; - async function teardown() { + const teardown = async () => { await security.testUser.restoreDefaults(); - } + }; - describe('Options list control', function () { + describe('Options list control', async () => { before(setup); after(teardown); @@ -46,5 +42,7 @@ export default function ({ loadTestFile, getService, getPageObjects }: FtrProvid loadTestFile(require.resolve('./options_list_dashboard_interaction')); loadTestFile(require.resolve('./options_list_suggestions')); loadTestFile(require.resolve('./options_list_validation')); + + loadTestFile(require.resolve('./options_list_allow_expensive_queries_off.ts')); }); } diff --git a/test/functional/apps/dashboard_elements/controls/options_list/options_list_allow_expensive_queries_off.ts b/test/functional/apps/dashboard_elements/controls/options_list/options_list_allow_expensive_queries_off.ts new file mode 100644 index 0000000000000..84af82f349713 --- /dev/null +++ b/test/functional/apps/dashboard_elements/controls/options_list/options_list_allow_expensive_queries_off.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { OPTIONS_LIST_CONTROL } from '@kbn/controls-plugin/common'; +import expect from '@kbn/expect'; + +import { OPTIONS_LIST_ANIMAL_SOUND_SUGGESTIONS } from '../../../../page_objects/dashboard_page_controls'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const browser = getService('browser'); + + const { dashboardControls, timePicker, console, common, dashboard, header } = getPageObjects([ + 'dashboardControls', + 'timePicker', + 'dashboard', + 'console', + 'common', + 'header', + ]); + + const setAllowExpensiveQueries = async (value: boolean) => { + await common.navigateToApp('console'); + await console.closeHelpIfExists(); + await console.clearTextArea(); + await console.enterRequest( + '\nPUT _cluster/settings\n{"transient": {"search.allow_expensive_queries": ' + value + '}}' + ); + await console.clickPlay(); + await header.waitUntilLoadingHasFinished(); + await browser.refresh(); + }; + + describe('Allow expensive queries setting is off', () => { + let controlId: string; + + before(async () => { + await setAllowExpensiveQueries(false); + + await common.navigateToApp('dashboard'); + await dashboard.clickNewDashboard(); + await dashboard.ensureDashboardIsInEditMode(); + await timePicker.setDefaultDataRange(); + await header.waitUntilLoadingHasFinished(); + + await dashboardControls.createControl({ + controlType: OPTIONS_LIST_CONTROL, + dataViewTitle: 'animals-*', + fieldName: 'sound.keyword', + }); + controlId = (await dashboardControls.getAllControlIds())[0]; + }); + + after(async () => { + await dashboard.gotoDashboardLandingPage(); + await dashboard.clickUnsavedChangesDiscard(`discard-unsaved-New-Dashboard`); + await setAllowExpensiveQueries(true); + }); + + it('Shows available options in options list', async () => { + await dashboardControls.ensureAvailableOptionsEqual(controlId, { + suggestions: OPTIONS_LIST_ANIMAL_SOUND_SUGGESTIONS, + invalidSelections: [], + }); + }); + + it('Can search options list for available options', async () => { + await dashboardControls.optionsListOpenPopover(controlId); + await dashboardControls.optionsListPopoverSearchForOption('meo'); + await dashboardControls.ensureAvailableOptionsEqual( + controlId, + { + suggestions: { meow: OPTIONS_LIST_ANIMAL_SOUND_SUGGESTIONS.meow }, + invalidSelections: [], + }, + true + ); + await dashboardControls.optionsListPopoverClearSearch(); + await dashboardControls.optionsListEnsurePopoverIsClosed(controlId); + }); + + it('Can search options list for available options - case sensitive', async () => { + await dashboardControls.optionsListOpenPopover(controlId); + await dashboardControls.optionsListPopoverSearchForOption('MEO'); + const cardinality = await dashboardControls.optionsListPopoverGetAvailableOptionsCount(); + expect(cardinality).to.be(0); + await dashboardControls.optionsListPopoverClearSearch(); + await dashboardControls.optionsListEnsurePopoverIsClosed(controlId); + }); + }); +} diff --git a/test/functional/page_objects/dashboard_page_controls.ts b/test/functional/page_objects/dashboard_page_controls.ts index 85908bce3d35c..c3160e650c2a8 100644 --- a/test/functional/page_objects/dashboard_page_controls.ts +++ b/test/functional/page_objects/dashboard_page_controls.ts @@ -428,9 +428,22 @@ export class DashboardPageControls extends FtrService { await this.retry.try(async () => { expect(await this.optionsListPopoverGetAvailableOptions()).to.eql(expectation); }); + if (await this.testSubjects.exists('optionsList-cardinality-label')) { + expect(await this.optionsListGetCardinalityValue()).to.be( + Object.keys(expectation.suggestions).length.toLocaleString() + ); + } if (!skipOpen) await this.optionsListEnsurePopoverIsClosed(controlId); } + public async optionsListGetCardinalityValue() { + this.log.debug(`getting the value of the cardinality badge`); + const cardinalityLabel = await ( + await this.testSubjects.find('optionsList-cardinality-label') + ).getVisibleText(); + return cardinalityLabel.split(' ')[0]; + } + public async optionsListPopoverSearchForOption(search: string) { this.log.debug(`searching for ${search} in options list`); await this.optionsListPopoverAssertOpen(); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 2a85faa2eb9e4..69a2dfeda1909 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -352,9 +352,7 @@ "controls.optionsList.errors.dataViewNotFound": "Impossible de localiser la vue de données : {dataViewId}", "controls.optionsList.errors.fieldNotFound": "Impossible de localiser le champ : {fieldName}", "controls.optionsList.popover.ariaLabel": "Fenêtre contextuelle pour le contrôle {fieldName}", - "controls.optionsList.popover.cardinalityPlaceholder": "Rechercher {totalOptions} {totalOptions, plural, one {option disponible} other {options disponibles}}", "controls.optionsList.popover.invalidSelectionsSectionTitle": "{invalidSelectionCount, plural, one {Sélection ignorée} other {Sélections ignorées}}", - "controls.optionsList.popover.invalidSelectionsTooltip": "{selectedOptions} {selectedOptions, plural, one {option sélectionnée} other {options sélectionnées}} {selectedOptions, plural, one {est ignorée} other {sont ignorées}}, car {selectedOptions, plural, one {elle n'est plus présente} other {elles ne sont plus présentes}} dans les données.", "controls.rangeSlider.errors.dataViewNotFound": "Impossible de localiser la vue de données : {dataViewId}", "controls.rangeSlider.errors.fieldNotFound": "Impossible de localiser le champ : {fieldName}", "controls.controlGroup.emptyState.addControlButtonTitle": "Ajouter un contrôle", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4e61b0224cc7b..aafb384cf9400 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -354,9 +354,7 @@ "controls.optionsList.errors.dataViewNotFound": "データビュー{dataViewId}が見つかりませんでした", "controls.optionsList.errors.fieldNotFound": "フィールド{fieldName}が見つかりませんでした", "controls.optionsList.popover.ariaLabel": "{fieldName}コントロールのポップオーバー", - "controls.optionsList.popover.cardinalityPlaceholder": "{totalOptions}個の使用可能な{totalOptions, plural, other {オプション}}を検索", "controls.optionsList.popover.invalidSelectionsSectionTitle": "{invalidSelectionCount, plural, other {個の選択項目}}", - "controls.optionsList.popover.invalidSelectionsTooltip": "{selectedOptions}個の選択した{selectedOptions, plural, other {オプション}} {selectedOptions, plural, other {が}}無視されます。{selectedOptions, plural, other {オプションが}}データに存在しません。", "controls.rangeSlider.errors.dataViewNotFound": "データビュー{dataViewId}が見つかりませんでした", "controls.rangeSlider.errors.fieldNotFound": "フィールド{fieldName}が見つかりませんでした", "controls.controlGroup.emptyState.addControlButtonTitle": "コントロールを追加", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index d581efbba60fe..ea907ebb37ab9 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -354,9 +354,7 @@ "controls.optionsList.errors.dataViewNotFound": "找不到数据视图:{dataViewId}", "controls.optionsList.errors.fieldNotFound": "找不到字段:{fieldName}", "controls.optionsList.popover.ariaLabel": "{fieldName} 控件的弹出框", - "controls.optionsList.popover.cardinalityPlaceholder": "搜索 {totalOptions} 个可用{totalOptions, plural, other {选项}}", "controls.optionsList.popover.invalidSelectionsSectionTitle": "已忽略{invalidSelectionCount, plural, other {个选择}}", - "controls.optionsList.popover.invalidSelectionsTooltip": "{selectedOptions} 个选定{selectedOptions, plural, other {选项}} {selectedOptions, plural, other {已}}忽略,因为{selectedOptions, plural, one {其} other {它们}}已不再在数据中。", "controls.rangeSlider.errors.dataViewNotFound": "找不到数据视图:{dataViewId}", "controls.rangeSlider.errors.fieldNotFound": "找不到字段:{fieldName}", "controls.controlGroup.emptyState.addControlButtonTitle": "添加控件", From 1ee97e1657e657f92d77581b09631ab936321717 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 6 Feb 2023 19:50:42 -0700 Subject: [PATCH 026/134] [kbn/babel-register] improve cache performance (#150261) After https://github.com/elastic/kibana/pull/146212 it feels like the babel-register cache is getting invalidated more frequently for some reason. The current version of the cache only stores a single cache entry for each file path, which shouldn't be too big of a problem but with these changes several versions of a file will be cached. The performance seems about equal, but because the cache contains multiple versions of a single file we should spend less time transpiling files when switching branches often. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- package.json | 1 + packages/kbn-babel-register/BUILD.bazel | 1 + packages/kbn-babel-register/cache/index.js | 5 +- .../kbn-babel-register/cache/lmdb_cache.js | 212 ++++++------------ .../cache/lmdb_cache.test.ts | 43 ++-- .../cache/no_cache_cache.js | 4 + packages/kbn-babel-register/cache/types.ts | 12 +- packages/kbn-babel-register/index.js | 14 +- .../kbn-babel-register/transforms/babel.js | 17 +- .../kbn-babel-register/transforms/peggy.js | 20 +- packages/kbn-babel-register/tsconfig.json | 1 - packages/kbn-cli-dev-mode/src/watcher.ts | 2 +- yarn.lock | 5 + 13 files changed, 127 insertions(+), 210 deletions(-) diff --git a/package.json b/package.json index d26b6bd79b0c2..78fa43cc621e1 100644 --- a/package.json +++ b/package.json @@ -1051,6 +1051,7 @@ "cypress-react-selector": "^3.0.0", "cypress-real-events": "^1.7.6", "cypress-recurse": "^1.26.0", + "date-fns": "^2.29.3", "debug": "^2.6.9", "delete-empty": "^2.0.0", "dependency-check": "^4.1.0", diff --git a/packages/kbn-babel-register/BUILD.bazel b/packages/kbn-babel-register/BUILD.bazel index a151c4e54c613..f97499a481baa 100644 --- a/packages/kbn-babel-register/BUILD.bazel +++ b/packages/kbn-babel-register/BUILD.bazel @@ -35,6 +35,7 @@ BUNDLER_DEPS = [ "@npm//chalk", "@npm//pirates", "@npm//lmdb", + "@npm//date-fns", "@npm//source-map-support", "//packages/kbn-repo-packages", "//packages/kbn-repo-info", diff --git a/packages/kbn-babel-register/cache/index.js b/packages/kbn-babel-register/cache/index.js index 7838a6c89028b..ea03e1396361e 100644 --- a/packages/kbn-babel-register/cache/index.js +++ b/packages/kbn-babel-register/cache/index.js @@ -10,7 +10,6 @@ const Fs = require('fs'); const Path = require('path'); const Crypto = require('crypto'); -const { readHashOfPackageMap } = require('@kbn/repo-packages'); const babel = require('@babel/core'); const peggy = require('@kbn/peggy'); const { REPO_ROOT, UPSTREAM_BRANCH } = require('@kbn/repo-info'); @@ -25,7 +24,6 @@ const { getBabelOptions } = require('@kbn/babel-transform'); */ function determineCachePrefix() { const json = JSON.stringify({ - synthPkgMapHash: readHashOfPackageMap(), babelVersion: babel.version, peggyVersion: peggy.version, // get a config for a fake js, ts, and tsx file to make sure we @@ -63,8 +61,7 @@ function getCache() { if (lmdbAvailable()) { log?.write('lmdb is available, using lmdb cache\n'); return new (require('./lmdb_cache').LmdbCache)({ - pathRoot: REPO_ROOT, - dir: Path.resolve(REPO_ROOT, 'data/babel_register_cache_v1', UPSTREAM_BRANCH), + dir: Path.resolve(REPO_ROOT, 'data/babel_register_cache', UPSTREAM_BRANCH), prefix: determineCachePrefix(), log, }); diff --git a/packages/kbn-babel-register/cache/lmdb_cache.js b/packages/kbn-babel-register/cache/lmdb_cache.js index 1c69ba1ed12a9..fe2179a591378 100644 --- a/packages/kbn-babel-register/cache/lmdb_cache.js +++ b/packages/kbn-babel-register/cache/lmdb_cache.js @@ -7,17 +7,21 @@ */ const Path = require('path'); +const Crypto = require('crypto'); +const startOfDay = /** @type {import('date-fns/startOfDay').default} */ ( + /** @type {unknown} */ (require('date-fns/startOfDay')) +); const chalk = require('chalk'); const LmdbStore = require('lmdb'); -const GLOBAL_ATIME = `${Date.now()}`; +const GLOBAL_ATIME = startOfDay(new Date()).valueOf(); const MINUTE = 1000 * 60; const HOUR = MINUTE * 60; const DAY = HOUR * 24; /** @typedef {import('./types').Cache} CacheInterface */ -/** @typedef {import('lmdb').Database} Db */ +/** @typedef {import('lmdb').Database} Db */ /** * @param {Db} db @@ -31,63 +35,29 @@ const dbName = (db) => * @implements {CacheInterface} */ class LmdbCache { - /** @type {import('lmdb').RootDatabase} */ - #codes; - /** @type {Db} */ - #atimes; - /** @type {Db} */ - #mtimes; - /** @type {Db} */ - #sourceMaps; - /** @type {string} */ - #pathRoot; - /** @type {string} */ - #prefix; + /** @type {import('lmdb').RootDatabase} */ + #db; /** @type {import('stream').Writable | undefined} */ #log; - /** @type {ReturnType} */ - #timer; + /** @type {string} */ + #prefix; /** * @param {import('./types').CacheConfig} config */ constructor(config) { - if (!Path.isAbsolute(config.pathRoot)) { - throw new Error('cache requires an absolute path to resolve paths relative to'); - } - - this.#pathRoot = config.pathRoot; - this.#prefix = config.prefix; this.#log = config.log; - - this.#codes = LmdbStore.open(config.dir, { - name: 'codes', - encoding: 'string', - maxReaders: 500, - }); - - // TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix - this.#atimes = this.#codes.openDB('atimes', { - name: 'atimes', - encoding: 'string', - }); - - this.#mtimes = this.#codes.openDB('mtimes', { - name: 'mtimes', - encoding: 'string', - }); - - this.#sourceMaps = this.#codes.openDB('sourceMaps', { - name: 'sourceMaps', - encoding: 'string', + this.#prefix = config.prefix; + this.#db = LmdbStore.open(Path.resolve(config.dir, 'v5'), { + name: 'db', + encoding: 'json', }); - // after the process has been running for 30 minutes prune the - // keys which haven't been used in 30 days. We use `unref()` to - // make sure this timer doesn't hold other processes open - // unexpectedly - this.#timer = setTimeout(() => { - this.#pruneOldKeys().catch((error) => { + const lastClean = this.#db.get('@last clean'); + if (!lastClean || lastClean[0] < GLOBAL_ATIME - 7 * DAY) { + try { + this.#pruneOldKeys(); + } catch (error) { process.stderr.write(` Failed to cleanup @kbn/babel-register cache: @@ -95,83 +65,60 @@ Failed to cleanup @kbn/babel-register cache: To eliminate this problem you may want to delete the "${Path.relative(process.cwd(), config.dir)}" directory and report this error to the Operations team.\n`); - }); - }, 30 * MINUTE); - - // timer.unref is not defined in jest which emulates the dom by default - if (typeof this.#timer.unref === 'function') { - this.#timer.unref(); + } finally { + this.#db.putSync('@last clean', [GLOBAL_ATIME, '', {}]); + } } } /** + * Get the cache key of the path and source from disk of a file * @param {string} path + * @param {string} source + * @returns {string} */ - getMtime(path) { - return this.#safeGet(this.#mtimes, this.#getKey(path)); + getKey(path, source) { + return `${this.#prefix}:${Crypto.createHash('sha1').update(path).update(source).digest('hex')}`; } /** - * @param {string} path + * @param {string} key + * @returns {string|undefined} */ - getCode(path) { - const key = this.#getKey(path); - const code = this.#safeGet(this.#codes, key); + getCode(key) { + const entry = this.#safeGet(this.#db, key); - if (code !== undefined) { + if (entry !== undefined && entry[0] !== GLOBAL_ATIME) { // when we use a file from the cache set the "atime" of that cache entry // so that we know which cache items we use and which haven't been - // touched in a long time (currently 30 days) - this.#safePut(this.#atimes, key, GLOBAL_ATIME); + // used in a long time (currently 30 days) + this.#safePut(this.#db, key, [GLOBAL_ATIME, entry[1], entry[2]]); } - return code; + return entry?.[1]; } /** - * @param {string} path - */ - getSourceMap(path) { - const map = this.#safeGet(this.#sourceMaps, this.#getKey(path)); - if (typeof map === 'string') { - return JSON.parse(map); - } - } - - close() { - clearTimeout(this.#timer); - } - - /** - * @param {string} path - * @param {{ mtime: string; code: string; map?: any }} file + * @param {string} key + * @returns {object|undefined} */ - async update(path, file) { - const key = this.#getKey(path); - - this.#safePut(this.#atimes, key, GLOBAL_ATIME); - this.#safePut(this.#mtimes, key, file.mtime); - this.#safePut(this.#codes, key, file.code); - - if (file.map) { - this.#safePut(this.#sourceMaps, key, JSON.stringify(file.map)); + getSourceMap(key) { + const entry = this.#safeGet(this.#db, key); + if (entry) { + return entry[2]; } } /** - * @param {string} path + * @param {string} key + * @param {{ code: string, map: object }} entry */ - #getKey(path) { - const normalizedPath = - Path.sep !== '/' - ? Path.relative(this.#pathRoot, path).split(Path.sep).join('/') - : Path.relative(this.#pathRoot, path); - - return `${this.#prefix}:${normalizedPath}`; + async update(key, entry) { + this.#safePut(this.#db, key, [GLOBAL_ATIME, entry.code, entry.map]); } /** - * @param {LmdbStore.Database} db + * @param {Db} db * @param {string} key */ #safeGet(db, key) { @@ -190,9 +137,9 @@ directory and report this error to the Operations team.\n`); } /** - * @param {LmdbStore.Database} db + * @param {Db} db * @param {string} key - * @param {string} value + * @param {import('./types').CacheEntry} value */ #safePut(db, key, value) { try { @@ -205,7 +152,7 @@ directory and report this error to the Operations team.\n`); /** * @param {string} type - * @param {LmdbStore.Database} db + * @param {Db} db * @param {string} key */ #debug(type, db, key) { @@ -214,7 +161,7 @@ directory and report this error to the Operations team.\n`); /** * @param {'GET' | 'PUT'} type - * @param {LmdbStore.Database} db + * @param {Db} db * @param {string} key * @param {Error} error */ @@ -227,51 +174,36 @@ directory and report this error to the Operations team.\n`); ); } - async #pruneOldKeys() { - try { - const ATIME_LIMIT = Date.now() - 30 * DAY; - const BATCH_SIZE = 1000; + #pruneOldKeys() { + const ATIME_LIMIT = Date.now() - 30 * DAY; - /** @type {string[]} */ - const validKeys = []; - /** @type {string[]} */ - const invalidKeys = []; + /** @type {string[]} */ + const toDelete = []; + const flushDeletes = () => { + if (!toDelete.length) { + return; + } - for (const { key, value } of this.#atimes.getRange()) { - const atime = parseInt(`${value}`, 10); - if (Number.isNaN(atime) || atime < ATIME_LIMIT) { - invalidKeys.push(key); - } else { - validKeys.push(key); + this.#db.transactionSync(() => { + for (const key of toDelete) { + this.#db.removeSync(key); } + }); + }; - if (validKeys.length + invalidKeys.length >= BATCH_SIZE) { - const promises = new Set(); - - if (invalidKeys.length) { - for (const k of invalidKeys) { - // all these promises are the same currently, so Set() will - // optimise this to a single promise, but I wouldn't be shocked - // if a future version starts returning independent promises so - // this is just for some future-proofing - promises.add(this.#atimes.remove(k)); - promises.add(this.#mtimes.remove(k)); - promises.add(this.#codes.remove(k)); - promises.add(this.#sourceMaps.remove(k)); - } - } else { - // delay a smidge to allow other things to happen before the next batch of checks - promises.add(new Promise((resolve) => setTimeout(resolve, 1))); - } + for (const { key, value } of this.#db.getRange()) { + if (Number.isNaN(value[0]) || value[0] < ATIME_LIMIT) { + toDelete.push(key); - invalidKeys.length = 0; - validKeys.length = 0; - await Promise.all(Array.from(promises)); + // flush deletes early if there are many deleted + if (toDelete.length > 10_000) { + flushDeletes(); } } - } catch { - // ignore errors, the cache is totally disposable and will rebuild if there is some sort of corruption } + + // delete all the old keys + flushDeletes(); } } diff --git a/packages/kbn-babel-register/cache/lmdb_cache.test.ts b/packages/kbn-babel-register/cache/lmdb_cache.test.ts index d752e45879aec..e1a8401ea9460 100644 --- a/packages/kbn-babel-register/cache/lmdb_cache.test.ts +++ b/packages/kbn-babel-register/cache/lmdb_cache.test.ts @@ -18,7 +18,7 @@ const DIR = Path.resolve(__dirname, '../__tmp__/cache'); const makeTestLog = () => { const log = Object.assign( new Writable({ - write(chunk, enc, cb) { + write(chunk, _, cb) { log.output += chunk; cb(); }, @@ -39,50 +39,35 @@ const makeCache = (...options: ConstructorParameters) => { }; beforeEach(async () => await del(DIR)); -afterEach(async () => { - await del(DIR); - for (const instance of instances) { - instance.close(); - } - instances.length = 0; -}); +afterEach(async () => await del(DIR)); it('returns undefined until values are set', async () => { const path = '/foo/bar.js'; - const mtime = new Date().toJSON(); + const source = `console.log("hi, hello")`; const log = makeTestLog(); const cache = makeCache({ dir: DIR, prefix: 'prefix', log, - pathRoot: '/foo/', }); - expect(cache.getMtime(path)).toBe(undefined); - expect(cache.getCode(path)).toBe(undefined); - expect(cache.getSourceMap(path)).toBe(undefined); + const key = cache.getKey(path, source); + expect(cache.getCode(key)).toBe(undefined); + expect(cache.getSourceMap(key)).toBe(undefined); - await cache.update(path, { - mtime, + await cache.update(key, { code: 'var x = 1', map: { foo: 'bar' }, }); - expect(cache.getMtime(path)).toBe(mtime); - expect(cache.getCode(path)).toBe('var x = 1'); - expect(cache.getSourceMap(path)).toEqual({ foo: 'bar' }); + expect(cache.getCode(key)).toBe('var x = 1'); + expect(cache.getSourceMap(key)).toEqual({ foo: 'bar' }); expect(log.output).toMatchInlineSnapshot(` - "MISS [mtimes] prefix:bar.js - MISS [codes] prefix:bar.js - MISS [sourceMaps] prefix:bar.js - PUT [atimes] prefix:bar.js - PUT [mtimes] prefix:bar.js - PUT [codes] prefix:bar.js - PUT [sourceMaps] prefix:bar.js - HIT [mtimes] prefix:bar.js - HIT [codes] prefix:bar.js - PUT [atimes] prefix:bar.js - HIT [sourceMaps] prefix:bar.js + "MISS [db] prefix:05a4b8198c4ec215d54d94681ef00ca9ecb45931 + MISS [db] prefix:05a4b8198c4ec215d54d94681ef00ca9ecb45931 + PUT [db] prefix:05a4b8198c4ec215d54d94681ef00ca9ecb45931 + HIT [db] prefix:05a4b8198c4ec215d54d94681ef00ca9ecb45931 + HIT [db] prefix:05a4b8198c4ec215d54d94681ef00ca9ecb45931 " `); }); diff --git a/packages/kbn-babel-register/cache/no_cache_cache.js b/packages/kbn-babel-register/cache/no_cache_cache.js index b4608e866d3b6..5d5aac93af0f2 100644 --- a/packages/kbn-babel-register/cache/no_cache_cache.js +++ b/packages/kbn-babel-register/cache/no_cache_cache.js @@ -12,6 +12,10 @@ * @implements {CacheInterface} */ class NoCacheCache { + getKey() { + return ''; + } + getCode() { return undefined; } diff --git a/packages/kbn-babel-register/cache/types.ts b/packages/kbn-babel-register/cache/types.ts index 6438662ae2d65..5189d3e7f3918 100644 --- a/packages/kbn-babel-register/cache/types.ts +++ b/packages/kbn-babel-register/cache/types.ts @@ -9,16 +9,16 @@ import { Writable } from 'stream'; export interface CacheConfig { - pathRoot: string; dir: string; prefix: string; log?: Writable; } export interface Cache { - getMtime(path: string): string | undefined; - getCode(path: string): string | undefined; - getSourceMap(path: string): object | undefined; - update(path: string, opts: { mtime: string; code: string; map?: any }): Promise; - close(): void; + getKey(path: string, source: string): string; + getCode(key: string): string | undefined; + getSourceMap(key: string): object | undefined; + update(key: string, entry: { code: string; map?: object | null }): Promise; } + +export type CacheEntry = [atime: number, code: string, sourceMap: object]; diff --git a/packages/kbn-babel-register/index.js b/packages/kbn-babel-register/index.js index ba20e1f1b18f0..457b0895919a4 100644 --- a/packages/kbn-babel-register/index.js +++ b/packages/kbn-babel-register/index.js @@ -41,6 +41,7 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +const Fs = require('fs'); const Path = require('path'); const { addHook } = require('pirates'); @@ -105,7 +106,18 @@ function install(options = undefined) { environment: 'node', // @ts-expect-error bad source-map-support types retrieveSourceMap(path) { - const map = cache.getSourceMap(path); + if (!Path.isAbsolute(path)) { + return null; + } + + let source; + try { + source = Fs.readFileSync(path, 'utf8'); + } catch { + return null; + } + + const map = cache.getSourceMap(cache.getKey(path, source)); return map ? { map, url: null } : null; }, }); diff --git a/packages/kbn-babel-register/transforms/babel.js b/packages/kbn-babel-register/transforms/babel.js index a1557bd528896..8328a60d477bb 100644 --- a/packages/kbn-babel-register/transforms/babel.js +++ b/packages/kbn-babel-register/transforms/babel.js @@ -6,25 +6,18 @@ * Side Public License, v 1. */ -const Fs = require('fs'); - const { transformCode } = require('@kbn/babel-transform'); /** @type {import('./types').Transform} */ const babelTransform = (path, source, cache) => { - const mtime = `${Fs.statSync(path).mtimeMs}`; - - if (cache.getMtime(path) === mtime) { - const code = cache.getCode(path); - if (code) { - return code; - } + const key = cache.getKey(path, source); + const cached = cache.getCode(key); + if (cached) { + return cached; } const result = transformCode(path, source); - - cache.update(path, { - mtime, + cache.update(key, { code: result.code, map: result.map, }); diff --git a/packages/kbn-babel-register/transforms/peggy.js b/packages/kbn-babel-register/transforms/peggy.js index b87676ca03bc3..6df30526d36fb 100644 --- a/packages/kbn-babel-register/transforms/peggy.js +++ b/packages/kbn-babel-register/transforms/peggy.js @@ -6,27 +6,16 @@ * Side Public License, v 1. */ -const Fs = require('fs'); -const Crypto = require('crypto'); - const Peggy = require('@kbn/peggy'); /** @type {import('./types').Transform} */ const peggyTransform = (path, source, cache) => { const config = Peggy.findConfigFile(path); - const mtime = `${Fs.statSync(path).mtimeMs}`; - const key = !config - ? path - : `${path}.config.${Crypto.createHash('sha256') - .update(config.source) - .digest('hex') - .slice(0, 8)}`; + const key = cache.getKey(path, source); - if (cache.getMtime(key) === mtime) { - const code = cache.getCode(key); - if (code) { - return code; - } + const cached = cache.getCode(key); + if (cached) { + return cached; } const code = Peggy.getJsSourceSync({ @@ -40,7 +29,6 @@ const peggyTransform = (path, source, cache) => { cache.update(key, { code, - mtime, }); return code; diff --git a/packages/kbn-babel-register/tsconfig.json b/packages/kbn-babel-register/tsconfig.json index 09576d2904a81..96e50650eee66 100644 --- a/packages/kbn-babel-register/tsconfig.json +++ b/packages/kbn-babel-register/tsconfig.json @@ -16,7 +16,6 @@ "@kbn/repo-info", "@kbn/babel-transform", "@kbn/peggy", - "@kbn/repo-packages", ], "exclude": [ "target/**/*", diff --git a/packages/kbn-cli-dev-mode/src/watcher.ts b/packages/kbn-cli-dev-mode/src/watcher.ts index 660022b6fc571..e4ade054b473d 100644 --- a/packages/kbn-cli-dev-mode/src/watcher.ts +++ b/packages/kbn-cli-dev-mode/src/watcher.ts @@ -77,7 +77,7 @@ export class Watcher { // ignore changes in any devOnly package, these can't power the server so we can ignore them if (pkg?.devOnly) { - return; + return pkg.id === '@kbn/babel-register'; } const result = this.classifier.classify(event.path); diff --git a/yarn.lock b/yarn.lock index fa46dea2940e9..8b1d994f4f695 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12314,6 +12314,11 @@ date-fns@^1.27.2, date-fns@^1.30.1: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw== +date-fns@^2.29.3: + version "2.29.3" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.3.tgz#27402d2fc67eb442b511b70bbdf98e6411cd68a8" + integrity sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA== + date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" From 1d0b90bd1257f57cbd8981bd5d2b0de32b4603c9 Mon Sep 17 00:00:00 2001 From: Andrew Macri Date: Mon, 6 Feb 2023 21:10:43 -0700 Subject: [PATCH 027/134] [Security Solution] Data Quality dashboard (#150063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # [Security Solution] Data Quality dashboard ## Check ECS compatibility with just one click With just one click, the _Data Quality dashboard_ checks all the indices used by the Security Solution, (or anything else), for compatibility with the [Elastic Common Schema (ECS)](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) ![checking_data_quality](https://user-images.githubusercontent.com/4459398/215989195-2f5e2126-9ece-4df6-9742-284c73442962.gif) ## Create cases from results Create a single case containing all the results, or create cases for specific indices ![create_case_from_take_action](https://user-images.githubusercontent.com/4459398/215989342-4489cf68-69d1-4ac4-859c-d849c4778d68.gif) ## Interactive tabs put results in context Expand any index to reveal interactive tabs - Summary - Incompatible fields - Custom fields - ECS complaint fields - All fields ![tabs](https://user-images.githubusercontent.com/4459398/215989435-a363a9e5-8635-42d1-a0f7-5e0ddc6f9515.gif) ## Share comprehensive markdown reports Share markdown reports containing the same content as the dashboard ![markdown_report](https://user-images.githubusercontent.com/4459398/215989555-72c53ed8-99f9-4be7-9181-6b9f365a8f6e.gif) ### On page load When the Data Quality dashboard page loads, the alerts index, and any indices matching the selected `Data view` are displayed ![page_load](https://user-images.githubusercontent.com/4459398/215989957-3b4d52f1-eaa4-4d42-9e40-d556602b006b.png) Only `hot`, `warm`, or `unmanaged` indices are displayed by default Indices are not checked automatically when the dashboard loads Click either : - `Check all` to check all the indices on the page - The expand button to automatically check (just) one index, and instantly view results ### Check all When the `Check all` button is clicked - The `Check all` button changes to a `Cancel` button - The `Last checked: n
+ } + className={customAccordionClassName} + data-test-subj="grouping-accordion" + extraAction={extraAction} + forceState={forceState} + id={`group${level}-${groupFieldValue}`} + onToggle={onToggle} + paddingSize="m" + > + {renderChildComponent(groupFilters)} + + ); +}; + +export const GroupPanel = React.memo(GroupPanelComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/container/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/container/index.test.tsx new file mode 100644 index 0000000000000..f51b722af2dec --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/container/index.test.tsx @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fireEvent, render, within } from '@testing-library/react'; +import React from 'react'; +import { GroupingContainer } from '..'; +import { TestProviders } from '../../../mock'; +import { createGroupFilter } from '../accordion_panel/helpers'; + +const renderChildComponent = jest.fn(); +const takeActionItems = jest.fn(); +const rule1Name = 'Rule 1 name'; +const rule1Desc = 'Rule 1 description'; +const rule2Name = 'Rule 2 name'; +const rule2Desc = 'Rule 2 description'; + +const testProps = { + data: { + groupsNumber: { + value: 2, + }, + stackByMultipleFields0: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: [rule1Name, rule1Desc], + key_as_string: `${rule1Name}|${rule1Desc}`, + doc_count: 1, + hostsCountAggregation: { + value: 1, + }, + ruleTags: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + alertsCount: { + value: 1, + }, + severitiesSubAggregation: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'low', + doc_count: 1, + }, + ], + }, + countSeveritySubAggregation: { + value: 1, + }, + usersCountAggregation: { + value: 1, + }, + }, + { + key: [rule2Name, rule2Desc], + key_as_string: `${rule2Name}|${rule2Desc}`, + doc_count: 1, + hostsCountAggregation: { + value: 1, + }, + ruleTags: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + alertsCount: { + value: 1, + }, + severitiesSubAggregation: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'low', + doc_count: 1, + }, + ], + }, + countSeveritySubAggregation: { + value: 1, + }, + usersCountAggregation: { + value: 1, + }, + }, + ], + }, + alertsCount: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [ + { + key: 'siem', + doc_count: 2, + }, + ], + }, + }, + pagination: { + pageIndex: 0, + pageSize: 25, + onChangeItemsPerPage: jest.fn(), + onChangePage: jest.fn(), + }, + renderChildComponent, + selectedGroup: 'kibana.alert.rule.name', + takeActionItems, +}; + +describe('grouping container', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('Renders group counts when groupsNumber > 0', () => { + const { getByTestId, getAllByTestId, queryByTestId } = render( + + + + ); + expect(getByTestId('alert-count').textContent).toBe('2 alerts'); + expect(getByTestId('groups-count').textContent).toBe('2 groups'); + expect(getAllByTestId('grouping-accordion').length).toBe(2); + expect(queryByTestId('empty-results-panel')).not.toBeInTheDocument(); + }); + + it('Does not render group counts when groupsNumber = 0', () => { + const data = { + groupsNumber: { + value: 0, + }, + stackByMultipleFields0: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + alertsCount: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, + buckets: [], + }, + }; + const { getByTestId, queryByTestId } = render( + + + + ); + expect(queryByTestId('alert-count')).not.toBeInTheDocument(); + expect(queryByTestId('groups-count')).not.toBeInTheDocument(); + expect(queryByTestId('grouping-accordion')).not.toBeInTheDocument(); + expect(getByTestId('empty-results-panel')).toBeInTheDocument(); + }); + + it('Opens one group at a time when each group is clicked', () => { + const { getAllByTestId } = render( + + + + ); + const group1 = within(getAllByTestId('grouping-accordion')[0]).getAllByRole('button')[0]; + const group2 = within(getAllByTestId('grouping-accordion')[1]).getAllByRole('button')[0]; + fireEvent.click(group1); + expect(renderChildComponent).toHaveBeenNthCalledWith( + 1, + createGroupFilter(testProps.selectedGroup, rule1Name) + ); + fireEvent.click(group2); + expect(renderChildComponent).toHaveBeenNthCalledWith( + 2, + createGroupFilter(testProps.selectedGroup, rule2Name) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/container/index.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/container/index.tsx new file mode 100644 index 0000000000000..d23e220e3c567 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/container/index.tsx @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiTablePagination } from '@elastic/eui'; +import type { Filter } from '@kbn/es-query'; +import React, { useMemo, useState } from 'react'; +import { firstNonNullValue } from '../../../../../common/endpoint/models/ecs_safety_helpers'; +import { createGroupFilter } from '../accordion_panel/helpers'; +import { tableDefaults } from '../../../store/data_table/defaults'; +import { defaultUnit } from '../../toolbar/unit'; +import type { BadgeMetric, CustomMetric } from '../accordion_panel'; +import { GroupPanel } from '../accordion_panel'; +import { GroupStats } from '../accordion_panel/group_stats'; +import { EmptyGroupingComponent } from '../empty_resuls_panel'; +import { GroupingStyledContainer, GroupsUnitCount } from '../styles'; +import { GROUPS_UNIT } from '../translations'; +import type { GroupingTableAggregation, RawBucket } from '../types'; + +interface GroupingContainerProps { + badgeMetricStats?: (fieldBucket: RawBucket) => BadgeMetric[]; + customMetricStats?: (fieldBucket: RawBucket) => CustomMetric[]; + data: GroupingTableAggregation & + Record< + string, + { + value?: number | null; + buckets?: Array<{ + doc_count?: number | null; + }>; + } + >; + groupPanelRenderer?: (fieldBucket: RawBucket) => JSX.Element | undefined; + groupsSelector?: JSX.Element; + inspectButton?: JSX.Element; + pagination: { + pageIndex: number; + pageSize: number; + onChangeItemsPerPage: (itemsPerPageNumber: number) => void; + onChangePage: (pageNumber: number) => void; + }; + renderChildComponent: (groupFilter: Filter[]) => React.ReactNode; + selectedGroup: string; + takeActionItems: (groupFilters: Filter[]) => JSX.Element[]; + unit?: (n: number) => string; +} + +const GroupingContainerComponent = ({ + badgeMetricStats, + customMetricStats, + data, + groupPanelRenderer, + groupsSelector, + inspectButton, + pagination, + renderChildComponent, + selectedGroup, + takeActionItems, + unit = defaultUnit, +}: GroupingContainerProps) => { + const [trigger, setTrigger] = useState< + Record + >({}); + + const groupsNumber = data?.groupsNumber?.value ?? 0; + const unitCountText = useMemo(() => { + const count = + data?.alertsCount?.buckets && data?.alertsCount?.buckets.length > 0 + ? data?.alertsCount?.buckets[0].doc_count ?? 0 + : 0; + return `${count.toLocaleString()} ${unit && unit(count)}`; + }, [data?.alertsCount?.buckets, unit]); + + const unitGroupsCountText = useMemo( + () => `${groupsNumber.toLocaleString()} ${GROUPS_UNIT(groupsNumber)}`, + [groupsNumber] + ); + + const groupPanels = useMemo( + () => + data.stackByMultipleFields0?.buckets?.map((groupBucket) => { + const group = firstNonNullValue(groupBucket.key); + const groupKey = `group0-${group}`; + + return ( + + + } + forceState={(trigger[groupKey] && trigger[groupKey].state) ?? 'closed'} + groupBucket={groupBucket} + groupPanelRenderer={groupPanelRenderer && groupPanelRenderer(groupBucket)} + onToggleGroup={(isOpen) => { + setTrigger({ + // ...trigger, -> this change will keep only one group at a time expanded and one table displayed + [groupKey]: { + state: isOpen ? 'open' : 'closed', + selectedBucket: groupBucket, + }, + }); + }} + renderChildComponent={ + trigger[groupKey] && trigger[groupKey].state === 'open' + ? renderChildComponent + : () => null + } + selectedGroup={selectedGroup} + /> + + + ); + }), + [ + badgeMetricStats, + customMetricStats, + data.stackByMultipleFields0?.buckets, + groupPanelRenderer, + renderChildComponent, + selectedGroup, + takeActionItems, + trigger, + ] + ); + const pageCount = useMemo( + () => (groupsNumber && pagination.pageSize ? Math.ceil(groupsNumber / pagination.pageSize) : 1), + [groupsNumber, pagination.pageSize] + ); + return ( + <> + + + {groupsNumber > 0 ? ( + + + {unitCountText} + + + + {unitGroupsCountText} + + + + ) : null} + + + + {inspectButton && {inspectButton}} + {groupsSelector} + + + + + {groupsNumber > 0 ? ( + <> + {groupPanels} + + + + ) : ( + + )} + + + ); +}; + +export const GroupingContainer = React.memo(GroupingContainerComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/empty_resuls_panel.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/empty_resuls_panel.tsx new file mode 100644 index 0000000000000..b51cd14e84bab --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/empty_resuls_panel.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem, EuiImage, EuiPanel, EuiText, EuiTitle } from '@elastic/eui'; +import React from 'react'; +import type { CoreStart } from '@kbn/core/public'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; + +const panelStyle = { + maxWidth: 500, +}; + +const heights = { + tall: 490, + short: 250, +}; + +export const EmptyGroupingComponent: React.FC<{ height?: keyof typeof heights }> = ({ + height = 'tall', +}) => { + const { http } = useKibana().services; + + return ( + + + + + + + + +

+ +

+
+

+ +

+
+
+ + + +
+
+
+
+
+ ); +}; diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/custom_field_panel.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/custom_field_panel.tsx new file mode 100644 index 0000000000000..dc45df7d7bf2e --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/custom_field_panel.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiComboBoxOptionOption } from '@elastic/eui'; +import { EuiButton, EuiComboBox, EuiForm, EuiFormRow } from '@elastic/eui'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export interface GroupByOptions { + text: string; + field: string; +} + +interface Props { + onSubmit: (field: string) => void; + fields: FieldSpec[]; + currentOptions: GroupByOptions[]; +} + +interface SelectedOption { + label: string; +} + +const initialState = { + selectedOptions: [] as SelectedOption[], +}; + +type State = Readonly; + +export class CustomFieldPanel extends React.PureComponent { + public static displayName = 'CustomFieldPanel'; + public readonly state: State = initialState; + public render() { + const { fields, currentOptions } = this.props; + const options = fields + .filter( + (f) => + f.aggregatable && + f.type === 'string' && + !(currentOptions && currentOptions.some((o) => o.field === f.name)) + ) + .map((f) => ({ label: f.name })); + const isSubmitDisabled = !this.state.selectedOptions.length; + return ( +
+ + + + + + {i18n.translate('xpack.securitySolution.selector.grouping.label.add', { + defaultMessage: 'Add', + })} + + +
+ ); + } + private handleSubmit = () => { + this.props.onSubmit(this.state.selectedOptions[0].label); + }; + + private handleFieldSelection = (selectedOptions: SelectedOption[]) => { + this.setState({ selectedOptions }); + }; +} diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.test.tsx new file mode 100644 index 0000000000000..a587206572f9d --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.test.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fireEvent, render } from '@testing-library/react'; +import { TestProviders } from '../../../mock'; +import { GroupsSelector } from '..'; +import React from 'react'; + +const onGroupChange = jest.fn(); +const testProps = { + fields: [ + { + name: 'kibana.alert.rule.name', + searchable: true, + type: 'string', + aggregatable: true, + esTypes: ['keyword'], + }, + { + name: 'host.name', + searchable: true, + type: 'string', + aggregatable: true, + esTypes: ['keyword'], + }, + { + name: 'user.name', + searchable: true, + type: 'string', + aggregatable: true, + esTypes: ['keyword'], + }, + { + name: 'source.ip', + searchable: true, + type: 'ip', + aggregatable: true, + esTypes: ['ip'], + }, + ], + groupSelected: 'kibana.alert.rule.name', + onGroupChange, + options: [ + { + label: 'Rule name', + key: 'kibana.alert.rule.name', + }, + { + label: 'User name', + key: 'user.name', + }, + { + label: 'Host name', + key: 'host.name', + }, + { + label: 'Source IP', + key: 'source.ip', + }, + ], + title: 'Group alerts by', +}; +describe('group selector', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('Sets the selected group from the groupSelected prop', () => { + const { getByTestId } = render( + + + + ); + expect(getByTestId('group-selector-dropdown').textContent).toBe('Group alerts by: Rule name'); + }); + it('Presents correct option when group selector dropdown is clicked', () => { + const { getByTestId } = render( + + + + ); + fireEvent.click(getByTestId('group-selector-dropdown')); + [ + ...testProps.options, + { key: 'none', label: 'None' }, + { key: 'custom', label: 'Custom field' }, + ].forEach((o) => { + expect(getByTestId(`panel-${o.key}`).textContent).toBe(o.label); + }); + }); + it('Presents fields dropdown when custom field option is selected', () => { + const { getByTestId } = render( + + + + ); + fireEvent.click(getByTestId('group-selector-dropdown')); + fireEvent.click(getByTestId('panel-none')); + expect(onGroupChange).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.tsx new file mode 100644 index 0000000000000..389645bf549da --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/groups_selector/index.tsx @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + EuiContextMenuPanelDescriptor, + EuiContextMenuPanelItemDescriptor, +} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiBetaBadge, EuiPopover } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useCallback, useMemo, useState } from 'react'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; +import { CustomFieldPanel } from './custom_field_panel'; +import { GROUP_BY, TECHNICAL_PREVIEW } from '../translations'; +import { StyledContextMenu, StyledEuiButtonEmpty } from '../styles'; + +const none = i18n.translate('xpack.securitySolution.groupsSelector.noneGroupByOptionName', { + defaultMessage: 'None', +}); + +interface GroupSelectorProps { + fields: FieldSpec[]; + groupSelected: string; + onGroupChange: (groupSelection: string) => void; + options: Array<{ key: string; label: string }>; + title?: string; +} + +const GroupsSelectorComponent = ({ + fields, + groupSelected = 'none', + onGroupChange, + options, + title = '', +}: GroupSelectorProps) => { + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const panels: EuiContextMenuPanelDescriptor[] = useMemo( + () => [ + { + id: 'firstPanel', + items: [ + { + 'data-test-subj': 'panel-none', + name: none, + icon: groupSelected === 'none' ? 'check' : 'empty', + onClick: () => onGroupChange('none'), + }, + ...options.map((o) => ({ + 'data-test-subj': `panel-${o.key}`, + name: o.label, + onClick: () => onGroupChange(o.key), + icon: groupSelected === o.key ? 'check' : 'empty', + })), + { + 'data-test-subj': `panel-custom`, + name: i18n.translate('xpack.securitySolution.groupsSelector.customGroupByOptionName', { + defaultMessage: 'Custom field', + }), + icon: 'empty', + panel: 'customPanel', + }, + ], + }, + { + id: 'customPanel', + title: i18n.translate('xpack.securitySolution.groupsSelector.customGroupByPanelTitle', { + defaultMessage: 'Group By Custom Field', + }), + width: 685, + content: ( + ({ text: o.label, field: o.key }))} + onSubmit={(field: string) => { + onGroupChange(field); + }} + fields={fields} + /> + ), + }, + ], + [fields, groupSelected, onGroupChange, options] + ); + const selectedOption = useMemo( + () => options.filter((groupOption) => groupOption.key === groupSelected), + [groupSelected, options] + ); + + const onButtonClick = useCallback(() => setIsPopoverOpen((currentVal) => !currentVal), []); + const closePopover = useCallback(() => setIsPopoverOpen(false), []); + + const button = useMemo( + () => ( + 0 ? selectedOption[0].label : none + } + size="xs" + > + {`${title ?? GROUP_BY}: ${ + groupSelected !== 'none' && selectedOption.length > 0 ? selectedOption[0].label : none + }`} + + ), + [groupSelected, onButtonClick, selectedOption, title] + ); + + return ( + + + + + + + + + + + ); +}; + +export const GroupsSelector = React.memo(GroupsSelectorComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/index.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/index.tsx new file mode 100644 index 0000000000000..46b3002ba5c54 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/index.tsx @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { NONE_GROUP_KEY } from './types'; + +export * from './container'; +export * from './query'; +export * from './groups_selector'; +export * from './types'; + +export const isNoneGroup = (groupKey: string) => groupKey === NONE_GROUP_KEY; diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/query/index.ts b/x-pack/plugins/security_solution/public/common/components/grouping/query/index.ts new file mode 100644 index 0000000000000..8ea6a07f522f7 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/query/index.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash/fp'; +import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +/** The maximum number of items to render */ +export const DEFAULT_STACK_BY_FIELD0_SIZE = 10; +export const DEFAULT_STACK_BY_FIELD1_SIZE = 10; + +interface OptionalSubAggregation { + stackByMultipleFields1: { + multi_terms: { + terms: Array<{ + field: string; + }>; + }; + }; +} + +export interface CardinalitySubAggregation { + [category: string]: { + cardinality: { + field: string; + }; + }; +} + +export interface TermsSubAggregation { + [category: string]: { + terms: { + field: string; + exclude?: string[]; + }; + }; +} + +export const getOptionalSubAggregation = ({ + stackByMultipleFields1, + stackByMultipleFields1Size, + stackByMultipleFields1From = 0, + stackByMultipleFields1Sort, + additionalStatsAggregationsFields1, +}: { + stackByMultipleFields1: string[] | undefined; + stackByMultipleFields1Size: number; + stackByMultipleFields1From?: number; + stackByMultipleFields1Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>; + additionalStatsAggregationsFields1: Array; +}): OptionalSubAggregation | {} => + stackByMultipleFields1 != null && !isEmpty(stackByMultipleFields1) + ? { + stackByMultipleFields1: { + multi_terms: { + terms: stackByMultipleFields1.map((stackByMultipleField1) => ({ + field: stackByMultipleField1, + })), + }, + aggs: { + bucket_truncate: { + bucket_sort: { + sort: stackByMultipleFields1Sort, + from: stackByMultipleFields1From, + size: stackByMultipleFields1Size, + }, + }, + ...additionalStatsAggregationsFields1.reduce( + (aggObj, subAgg) => Object.assign(aggObj, subAgg), + {} + ), + }, + }, + } + : {}; + +export const getGroupingQuery = ({ + additionalFilters = [], + additionalAggregationsRoot, + additionalStatsAggregationsFields0, + additionalStatsAggregationsFields1, + from, + runtimeMappings, + stackByMultipleFields0, + stackByMultipleFields0Size = DEFAULT_STACK_BY_FIELD0_SIZE, + stackByMultipleFields0From, + stackByMultipleFields0Sort, + stackByMultipleFields1, + stackByMultipleFields1Size = DEFAULT_STACK_BY_FIELD1_SIZE, + stackByMultipleFields1From, + stackByMultipleFields1Sort, + to, +}: { + additionalFilters: Array<{ + bool: { filter: unknown[]; should: unknown[]; must_not: unknown[]; must: unknown[] }; + }>; + from: string; + runtimeMappings?: MappingRuntimeFields; + additionalAggregationsRoot?: Array; + stackByMultipleFields0: string[]; + stackByMultipleFields0Size?: number; + stackByMultipleFields0From?: number; + stackByMultipleFields0Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>; + additionalStatsAggregationsFields0: Array; + stackByMultipleFields1: string[] | undefined; + stackByMultipleFields1Size?: number; + stackByMultipleFields1From?: number; + stackByMultipleFields1Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>; + additionalStatsAggregationsFields1: Array; + to: string; +}) => ({ + size: 0, + aggs: { + stackByMultipleFields0: { + ...(stackByMultipleFields0.length > 1 + ? { + multi_terms: { + terms: stackByMultipleFields0.map((stackByMultipleField0) => ({ + field: stackByMultipleField0, + })), + }, + } + : { + terms: { + field: stackByMultipleFields0[0], + size: 10000, + }, + }), + aggs: { + ...getOptionalSubAggregation({ + stackByMultipleFields1, + stackByMultipleFields1Size, + stackByMultipleFields1From, + stackByMultipleFields1Sort, + additionalStatsAggregationsFields1, + }), + bucket_truncate: { + bucket_sort: { + sort: stackByMultipleFields0Sort, + from: stackByMultipleFields0From, + size: stackByMultipleFields0Size, + }, + }, + ...additionalStatsAggregationsFields0.reduce( + (aggObj, subAgg) => Object.assign(aggObj, subAgg), + {} + ), + }, + }, + ...(additionalAggregationsRoot + ? additionalAggregationsRoot.reduce((aggObj, subAgg) => Object.assign(aggObj, subAgg), {}) + : {}), + }, + query: { + bool: { + filter: [ + ...additionalFilters, + { + range: { + '@timestamp': { + gte: from, + lte: to, + }, + }, + }, + ], + }, + }, + runtime_mappings: runtimeMappings, + _source: false, +}); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/styles.tsx b/x-pack/plugins/security_solution/public/common/components/grouping/styles.tsx new file mode 100644 index 0000000000000..80eb0183cb510 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/styles.tsx @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButtonEmpty, EuiContextMenu } from '@elastic/eui'; +import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import styled from 'styled-components'; + +export const GroupsUnitCount = styled.span` + font-size: ${({ theme }) => theme.eui.euiFontSizeXS}; + font-weight: ${({ theme }) => theme.eui.euiFontWeightSemiBold}; + border-right: ${({ theme }) => theme.eui.euiBorderThin}; + margin-right: 16px; + padding-right: 16px; +`; + +export const StatsContainer = styled.span` + font-size: ${({ theme }) => theme.eui.euiFontSizeXS}; + font-weight: ${({ theme }) => theme.eui.euiFontWeightSemiBold}; + border-right: ${({ theme }) => theme.eui.euiBorderThin}; + margin-right: 16px; + padding-right: 16px; + .smallDot { + width: 3px !important; + display: inline-block; + } + .euiBadge__text { + text-align: center; + width: 100%; + } +`; + +export const GroupingStyledContainer = styled.div` + .euiAccordion__childWrapper .euiAccordion__padding--m { + margin-left: 8px; + margin-right: 8px; + border-left: ${({ theme }) => theme.eui.euiBorderThin}; + border-right: ${({ theme }) => theme.eui.euiBorderThin}; + border-bottom: ${({ theme }) => theme.eui.euiBorderThin}; + border-radius: 0 0 6px 6px; + } + .euiAccordion__triggerWrapper { + border-bottom: ${({ theme }) => theme.eui.euiBorderThin}; + border-left: ${({ theme }) => theme.eui.euiBorderThin}; + border-right: ${({ theme }) => theme.eui.euiBorderThin}; + border-radius: 6px; + min-height: 78px; + padding-left: 16px; + padding-right: 16px; + } + .groupingAccordionForm { + border-top: ${({ theme }) => theme.eui.euiBorderThin}; + border-bottom: none; + border-radius: 6px; + min-width: 1090px; + } + .groupingAccordionForm__button { + text-decoration: none !important; + } + .groupingPanelRenderer { + display: table; + table-layout: fixed; + width: 100%; + padding-right: 32px; + } +`; + +export const StyledContextMenu = euiStyled(EuiContextMenu)` + width: 250px; + & .euiContextMenuItem__text { + overflow: hidden; + text-overflow: ellipsis; + } + .euiContextMenuItem { + border-bottom: ${({ theme }) => theme.eui.euiBorderThin}; + } + .euiContextMenuItem:last-child { + border: none; + } +`; + +export const StyledEuiButtonEmpty = euiStyled(EuiButtonEmpty)` + font-weight: 'normal'; + + .euiButtonEmpty__text { + max-width: 300px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } +`; diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/translations.ts b/x-pack/plugins/security_solution/public/common/components/grouping/translations.ts new file mode 100644 index 0000000000000..2e73e95b80de5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/translations.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const GROUPS_UNIT = (totalCount: number) => + i18n.translate('xpack.securitySolution.grouping.total.unit', { + values: { totalCount }, + defaultMessage: `{totalCount, plural, =1 {group} other {groups}}`, + }); + +export const TAKE_ACTION = i18n.translate( + 'xpack.securitySolution.grouping.additionalActions.takeAction', + { + defaultMessage: 'Take actions', + } +); + +export const TECHNICAL_PREVIEW = i18n.translate( + 'xpack.securitySolution.grouping.technicalPreviewLabel', + { + defaultMessage: 'Technical Preview', + } +); + +export const GROUP_BY = i18n.translate('xpack.securitySolution.selector.grouping.label', { + defaultMessage: 'Group by field', +}); diff --git a/x-pack/plugins/security_solution/public/common/components/grouping/types.ts b/x-pack/plugins/security_solution/public/common/components/grouping/types.ts new file mode 100644 index 0000000000000..b4685c75cab29 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/grouping/types.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { GenericBuckets } from '../../../../common/search_strategy/common'; + +export const DEFAULT_GROUPING_QUERY_ID = 'defaultGroupingQuery'; + +export const NONE_GROUP_KEY = 'none'; + +export type RawBucket = GenericBuckets & { + alertsCount?: { + value?: number | null; // Elasticsearch returns `null` when a sub-aggregation cannot be computed + }; + severitiesSubAggregation?: { + buckets?: GenericBuckets[]; + }; + countSeveritySubAggregation?: { + value?: number | null; // Elasticsearch returns `null` when a sub-aggregation cannot be computed + }; + usersCountAggregation?: { + value?: number | null; // Elasticsearch returns `null` when a sub-aggregation cannot be computed + }; + hostsCountAggregation?: { + value?: number | null; // Elasticsearch returns `null` when a sub-aggregation cannot be computed + }; + rulesCountAggregation?: { + value?: number | null; // Elasticsearch returns `null` when a sub-aggregation cannot be computed + }; + ruleTags?: { + doc_count_error_upper_bound?: number; + sum_other_doc_count?: number; + buckets?: GenericBuckets[]; + }; + stackByMultipleFields1?: { + buckets?: GenericBuckets[]; + doc_count_error_upper_bound?: number; + sum_other_doc_count?: number; + }; +}; + +/** Defines the shape of the aggregation returned by Elasticsearch */ +export interface GroupingTableAggregation { + stackByMultipleFields0?: { + buckets?: RawBucket[]; + }; + groupsCount0?: { + value?: number | null; + }; +} + +export type GroupingFieldTotalAggregation = Record< + string, + { value?: number | null; buckets?: Array<{ doc_count?: number | null }> } +>; + +export type FlattenedBucket = Pick< + RawBucket, + 'doc_count' | 'key' | 'key_as_string' | 'alertsCount' +> & { + stackByMultipleFields1Key?: string; + stackByMultipleFields1DocCount?: number; +}; diff --git a/x-pack/plugins/security_solution/public/common/components/landing_cards/index.tsx b/x-pack/plugins/security_solution/public/common/components/landing_cards/index.tsx index 463f98feb2050..d145598faab9e 100644 --- a/x-pack/plugins/security_solution/public/common/components/landing_cards/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/landing_cards/index.tsx @@ -94,7 +94,6 @@ export const LandingCards = memo(() => {