From 100afab3085e735fa0624a07a54a7f752d92935f Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 10 Sep 2020 16:06:41 +0200 Subject: [PATCH 01/29] Use new date_histogram intervals in timelion (#77160) --- .../server/series_functions/es/es.test.js | 12 ++++++++++-- .../series_functions/es/lib/create_date_agg.js | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/plugins/vis_type_timelion/server/series_functions/es/es.test.js b/src/plugins/vis_type_timelion/server/series_functions/es/es.test.js index 4b5aab85cfc4e..c5fc4b7b93269 100644 --- a/src/plugins/vis_type_timelion/server/series_functions/es/es.test.js +++ b/src/plugins/vis_type_timelion/server/series_functions/es/es.test.js @@ -100,9 +100,17 @@ describe('es', () => { expect(agg.time_buckets.date_histogram.time_zone).to.equal('Etc/UTC'); }); - it('sets the field and interval', () => { + it('sets the field', () => { expect(agg.time_buckets.date_histogram.field).to.equal('@timestamp'); - expect(agg.time_buckets.date_histogram.interval).to.equal('1y'); + }); + + it('sets the interval for calendar_interval correctly', () => { + expect(agg.time_buckets.date_histogram).to.have.property('calendar_interval', '1y'); + }); + + it('sets the interval for fixed_interval correctly', () => { + const a = createDateAgg({ timefield: '@timestamp', interval: '24h' }, tlConfig); + expect(a.time_buckets.date_histogram).to.have.property('fixed_interval', '24h'); }); it('sets min_doc_count to 0', () => { diff --git a/src/plugins/vis_type_timelion/server/series_functions/es/lib/create_date_agg.js b/src/plugins/vis_type_timelion/server/series_functions/es/lib/create_date_agg.js index 904fe69cbc57c..b36f37ac5cc9d 100644 --- a/src/plugins/vis_type_timelion/server/series_functions/es/lib/create_date_agg.js +++ b/src/plugins/vis_type_timelion/server/series_functions/es/lib/create_date_agg.js @@ -19,6 +19,8 @@ import _ from 'lodash'; import { buildAggBody } from './agg_body'; +import { search } from '../../../../../../plugins/data/server'; +const { dateHistogramInterval } = search.aggs; export default function createDateAgg(config, tlConfig, scriptedFields) { const dateAgg = { @@ -26,13 +28,13 @@ export default function createDateAgg(config, tlConfig, scriptedFields) { meta: { type: 'time_buckets' }, date_histogram: { field: config.timefield, - interval: config.interval, time_zone: tlConfig.time.timezone, extended_bounds: { min: tlConfig.time.from, max: tlConfig.time.to, }, min_doc_count: 0, + ...dateHistogramInterval(config.interval), }, }, }; From fd0878277791e75604e06a284020438bd2a77bfc Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 10 Sep 2020 16:47:38 +0200 Subject: [PATCH 02/29] [CI] ensure tests for @elastic/safer-lodash-set only runs once (#77146) --- tasks/config/run.js | 6 ------ tasks/jenkins.js | 1 - test/scripts/test/safer_lodash_set.sh | 5 ----- vars/tasks.groovy | 1 - 4 files changed, 13 deletions(-) delete mode 100755 test/scripts/test/safer_lodash_set.sh diff --git a/tasks/config/run.js b/tasks/config/run.js index 132b51765b3ed..148be6ea8afaa 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -154,12 +154,6 @@ module.exports = function () { args: ['scripts/test_hardening.js'], }), - test_package_safer_lodash_set: scriptWithGithubChecks({ - title: '@elastic/safer-lodash-set tests', - cmd: YARN, - args: ['--cwd', 'packages/elastic-safer-lodash-set', 'test'], - }), - apiIntegrationTests: scriptWithGithubChecks({ title: 'API integration tests', cmd: NODE, diff --git a/tasks/jenkins.js b/tasks/jenkins.js index adfb6f0f46868..90efadf41c435 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -38,7 +38,6 @@ module.exports = function (grunt) { 'run:test_jest_integration', 'run:test_projects', 'run:test_hardening', - 'run:test_package_safer_lodash_set', 'run:apiIntegrationTests', ]); }; diff --git a/test/scripts/test/safer_lodash_set.sh b/test/scripts/test/safer_lodash_set.sh deleted file mode 100755 index 4d7f9c28210d1..0000000000000 --- a/test/scripts/test/safer_lodash_set.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -source src/dev/ci_setup/setup_env.sh - -yarn run grunt run:test_package_safer_lodash_set diff --git a/vars/tasks.groovy b/vars/tasks.groovy index 52641ce31f0be..edd2c0aa47401 100644 --- a/vars/tasks.groovy +++ b/vars/tasks.groovy @@ -34,7 +34,6 @@ def test() { kibanaPipeline.scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh'), kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh'), - kibanaPipeline.scriptTask('@elastic/safer-lodash-set Tests', 'test/scripts/test/safer_lodash_set.sh'), kibanaPipeline.scriptTask('X-Pack SIEM cyclic dependency', 'test/scripts/test/xpack_siem_cyclic_dependency.sh'), kibanaPipeline.scriptTask('X-Pack List cyclic dependency', 'test/scripts/test/xpack_list_cyclic_dependency.sh'), kibanaPipeline.scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh'), From ae9a9c2f7191fc92fb903244ac35035603727ccd Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 10 Sep 2020 16:06:39 +0100 Subject: [PATCH 03/29] [ML] Improve performance of job exists check (#77156) * [ML] Improve performance of job exists check * adding tests * possible undefined error body --- .../ml/server/models/job_service/jobs.ts | 31 ++-- .../apis/ml/jobs/jobs_exist.ts | 145 ++++++++++++++++++ 2 files changed, 157 insertions(+), 19 deletions(-) create mode 100644 x-pack/test/api_integration/apis/ml/jobs/jobs_exist.ts diff --git a/x-pack/plugins/ml/server/models/job_service/jobs.ts b/x-pack/plugins/ml/server/models/job_service/jobs.ts index e047d31ba6eb7..f4378e29ef826 100644 --- a/x-pack/plugins/ml/server/models/job_service/jobs.ts +++ b/x-pack/plugins/ml/server/models/job_service/jobs.ts @@ -407,28 +407,21 @@ export function jobsProvider(client: IScopedClusterClient) { // Job IDs in supplied array may contain wildcard '*' characters // e.g. *_low_request_rate_ecs async function jobsExist(jobIds: string[] = []) { - // Get the list of job IDs. - const { body } = await asInternalUser.ml.getJobs({ - job_id: jobIds.join(), - }); - const results: { [id: string]: boolean } = {}; - if (body.count > 0) { - const allJobIds = body.jobs.map((job) => job.job_id); - - // Check if each of the supplied IDs match existing jobs. - jobIds.forEach((jobId) => { - // Create a Regex for each supplied ID as wildcard * is allowed. - const regexp = new RegExp(`^${jobId.replace(/\*+/g, '.*')}$`); - const exists = allJobIds.some((existsJobId) => regexp.test(existsJobId)); - results[jobId] = exists; - }); - } else { - jobIds.forEach((jobId) => { + for (const jobId of jobIds) { + try { + const { body } = await asInternalUser.ml.getJobs({ + job_id: jobId, + }); + results[jobId] = body.count > 0; + } catch (e) { + // if a non-wildcarded job id is supplied, the get jobs endpoint will 404 + if (e.body?.status !== 404) { + throw e; + } results[jobId] = false; - }); + } } - return results; } diff --git a/x-pack/test/api_integration/apis/ml/jobs/jobs_exist.ts b/x-pack/test/api_integration/apis/ml/jobs/jobs_exist.ts new file mode 100644 index 0000000000000..c48376b6a14f3 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/jobs/jobs_exist.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { SINGLE_METRIC_JOB_CONFIG, DATAFEED_CONFIG } from './common_jobs'; + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const testSetupJobConfigs = [SINGLE_METRIC_JOB_CONFIG]; + + const responseBody = { + [SINGLE_METRIC_JOB_CONFIG.job_id]: true, + [`${SINGLE_METRIC_JOB_CONFIG.job_id.slice(0, 10)}*`]: true, // wildcard, use first 10 chars + [`${SINGLE_METRIC_JOB_CONFIG.job_id}_fail`]: false, + [`${SINGLE_METRIC_JOB_CONFIG.job_id.slice(0, 10)}_fail*`]: false, // wildcard, use first 10 chars + }; + + const testDataList = [ + { + testTitle: 'as ML Poweruser', + user: USER.ML_POWERUSER, + requestBody: { + jobIds: Object.keys(responseBody), + }, + expected: { + responseCode: 200, + responseBody, + }, + }, + { + testTitle: 'as ML Viewer', + user: USER.ML_VIEWER, + requestBody: { + jobIds: Object.keys(responseBody), + }, + expected: { + responseCode: 200, + responseBody, + }, + }, + ]; + + const testDataListUnauthorized = [ + { + testTitle: 'as ML Unauthorized user', + user: USER.ML_UNAUTHORIZED, + requestBody: { + jobIds: Object.keys(responseBody), + }, + expected: { + responseCode: 404, + error: 'Not Found', + }, + }, + ]; + + async function runJobsExistRequest( + user: USER, + requestBody: object, + expectedResponsecode: number + ): Promise { + const { body } = await supertest + .post('/api/ml/jobs/jobs_exist') + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(COMMON_REQUEST_HEADERS) + .send(requestBody) + .expect(expectedResponsecode); + + return body; + } + + describe('jobs_exist', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + after(async () => { + await ml.api.cleanMlIndices(); + }); + + it('sets up jobs', async () => { + for (const job of testSetupJobConfigs) { + const datafeedId = `datafeed-${job.job_id}`; + await ml.api.createAnomalyDetectionJob(job); + await ml.api.openAnomalyDetectionJob(job.job_id); + await ml.api.createDatafeed({ + ...DATAFEED_CONFIG, + datafeed_id: datafeedId, + job_id: job.job_id, + }); + } + }); + + describe('jobs exist', function () { + for (const testData of testDataList) { + it(`${testData.testTitle}`, async () => { + const body = await runJobsExistRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + const expectedResponse = testData.expected.responseBody; + const expectedRspJobIds = Object.keys(expectedResponse).sort((a, b) => + a.localeCompare(b) + ); + const actualRspJobIds = Object.keys(body).sort((a, b) => a.localeCompare(b)); + + expect(actualRspJobIds).to.have.length(expectedRspJobIds.length); + expect(actualRspJobIds).to.eql(expectedRspJobIds); + expectedRspJobIds.forEach((id) => { + expect(body[id]).to.eql(testData.expected.responseBody[id]); + }); + }); + } + }); + + describe('rejects request', function () { + for (const testData of testDataListUnauthorized) { + describe('fails to check jobs exist', function () { + it(`${testData.testTitle}`, async () => { + const body = await runJobsExistRequest( + testData.user, + testData.requestBody, + testData.expected.responseCode + ); + + expect(body).to.have.property('error').eql(testData.expected.error); + }); + }); + } + }); + }); +}; From e2cbd89e66693553743fa1a4896e9bdeb02bff26 Mon Sep 17 00:00:00 2001 From: CJ Cenizal Date: Thu, 10 Sep 2020 08:23:13 -0700 Subject: [PATCH 04/29] Rename useRequest's sendRequest return function to resendRequest and remove return value (#76795) --- .../request/use_request.test.helpers.tsx | 4 +-- .../public/request/use_request.test.ts | 22 +++++++------- .../public/request/use_request.ts | 30 ++++++++----------- .../components/node_allocation.tsx | 4 +-- .../components/node_attrs_details.tsx | 4 +-- .../components/snapshot_policies.tsx | 4 +-- .../edit_policy/edit_policy.container.tsx | 4 +-- .../policy_table/policy_table.container.tsx | 6 ++-- .../component_template_list.tsx | 8 ++--- .../data_stream_list/data_stream_list.tsx | 2 +- .../data_stream_table/data_stream_table.tsx | 4 +-- .../template_table/template_table.tsx | 4 +-- .../template_details_content.tsx | 4 +-- .../home/template_list/template_list.tsx | 2 +- .../template_table/template_table.tsx | 4 +-- .../index_management/public/shared_imports.ts | 1 + .../step_select_agent_policy.tsx | 2 +- .../details_page/hooks/use_agent_status.tsx | 2 +- .../sections/agent_policy/list_page/index.tsx | 10 +++---- .../sections/data_stream/list_page/index.tsx | 4 +-- .../components/agent_events_table.tsx | 4 +-- .../fleet/agent_details_page/index.tsx | 2 +- .../sections/fleet/agent_list_page/index.tsx | 4 +-- .../enrollment_token_list_page/index.tsx | 7 +++-- .../sections/pipelines_list/main.tsx | 8 ++--- .../policy_form/steps/step_logistics.tsx | 2 +- .../policy_details/policy_details.tsx | 2 +- .../sections/home/policy_list/policy_list.tsx | 4 +-- .../policy_list/policy_table/policy_table.tsx | 5 ++-- .../home/repository_list/repository_list.tsx | 2 +- .../repository_table/repository_table.tsx | 4 +-- .../home/restore_list/restore_list.tsx | 12 +++++--- .../home/snapshot_list/snapshot_list.tsx | 2 +- .../snapshot_table/snapshot_table.tsx | 5 ++-- .../snapshot_restore/public/shared_imports.ts | 1 + .../watch_visualization.tsx | 2 +- 36 files changed, 96 insertions(+), 95 deletions(-) diff --git a/src/plugins/es_ui_shared/public/request/use_request.test.helpers.tsx b/src/plugins/es_ui_shared/public/request/use_request.test.helpers.tsx index 0d6fd122ad22c..7a42ed7fad427 100644 --- a/src/plugins/es_ui_shared/public/request/use_request.test.helpers.tsx +++ b/src/plugins/es_ui_shared/public/request/use_request.test.helpers.tsx @@ -106,7 +106,7 @@ export const createUseRequestHelpers = (): UseRequestHelpers => { }; const TestComponent = ({ requestConfig }: { requestConfig: UseRequestConfig }) => { - const { isInitialRequest, isLoading, error, data, sendRequest } = useRequest( + const { isInitialRequest, isLoading, error, data, resendRequest } = useRequest( httpClient as HttpSetup, requestConfig ); @@ -115,7 +115,7 @@ export const createUseRequestHelpers = (): UseRequestHelpers => { hookResult.isLoading = isLoading; hookResult.error = error; hookResult.data = data; - hookResult.sendRequest = sendRequest; + hookResult.resendRequest = resendRequest; return null; }; diff --git a/src/plugins/es_ui_shared/public/request/use_request.test.ts b/src/plugins/es_ui_shared/public/request/use_request.test.ts index f7902218d9314..2a639f93b47b4 100644 --- a/src/plugins/es_ui_shared/public/request/use_request.test.ts +++ b/src/plugins/es_ui_shared/public/request/use_request.test.ts @@ -102,7 +102,7 @@ describe('useRequest hook', () => { setupSuccessRequest(); expect(hookResult.isInitialRequest).toBe(true); - hookResult.sendRequest(); + hookResult.resendRequest(); await completeRequest(); expect(hookResult.isInitialRequest).toBe(false); }); @@ -148,7 +148,7 @@ describe('useRequest hook', () => { expect(hookResult.error).toBe(getErrorResponse().error); act(() => { - hookResult.sendRequest(); + hookResult.resendRequest(); }); expect(hookResult.isLoading).toBe(true); expect(hookResult.error).toBe(getErrorResponse().error); @@ -183,7 +183,7 @@ describe('useRequest hook', () => { expect(hookResult.data).toBe(getSuccessResponse().data); act(() => { - hookResult.sendRequest(); + hookResult.resendRequest(); }); expect(hookResult.isLoading).toBe(true); expect(hookResult.data).toBe(getSuccessResponse().data); @@ -215,7 +215,7 @@ describe('useRequest hook', () => { }); describe('callbacks', () => { - describe('sendRequest', () => { + describe('resendRequest', () => { it('sends the request', async () => { const { setupSuccessRequest, completeRequest, hookResult, getSendRequestSpy } = helpers; setupSuccessRequest(); @@ -224,7 +224,7 @@ describe('useRequest hook', () => { expect(getSendRequestSpy().callCount).toBe(1); await act(async () => { - hookResult.sendRequest(); + hookResult.resendRequest(); await completeRequest(); }); expect(getSendRequestSpy().callCount).toBe(2); @@ -239,17 +239,17 @@ describe('useRequest hook', () => { await advanceTime(REQUEST_TIME); expect(getSendRequestSpy().callCount).toBe(1); act(() => { - hookResult.sendRequest(); + hookResult.resendRequest(); }); // The manual request resolves, and we'll send yet another one... await advanceTime(REQUEST_TIME); expect(getSendRequestSpy().callCount).toBe(2); act(() => { - hookResult.sendRequest(); + hookResult.resendRequest(); }); - // At this point, we've moved forward 3s. The poll is set at 2s. If sendRequest didn't + // At this point, we've moved forward 3s. The poll is set at 2s. If resendRequest didn't // reset the poll, the request call count would be 4, not 3. await advanceTime(REQUEST_TIME); expect(getSendRequestSpy().callCount).toBe(3); @@ -291,14 +291,14 @@ describe('useRequest hook', () => { const HALF_REQUEST_TIME = REQUEST_TIME * 0.5; setupSuccessRequest({ pollIntervalMs: REQUEST_TIME }); - // Before the original request resolves, we make a manual sendRequest call. + // Before the original request resolves, we make a manual resendRequest call. await advanceTime(HALF_REQUEST_TIME); expect(getSendRequestSpy().callCount).toBe(0); act(() => { - hookResult.sendRequest(); + hookResult.resendRequest(); }); - // The original quest resolves but it's been marked as outdated by the the manual sendRequest + // The original quest resolves but it's been marked as outdated by the the manual resendRequest // call "interrupts", so data is left undefined. await advanceTime(HALF_REQUEST_TIME); expect(getSendRequestSpy().callCount).toBe(1); diff --git a/src/plugins/es_ui_shared/public/request/use_request.ts b/src/plugins/es_ui_shared/public/request/use_request.ts index 481843bf40e88..e04f84a67b8a3 100644 --- a/src/plugins/es_ui_shared/public/request/use_request.ts +++ b/src/plugins/es_ui_shared/public/request/use_request.ts @@ -20,11 +20,7 @@ import { useEffect, useCallback, useState, useRef, useMemo } from 'react'; import { HttpSetup } from '../../../../../src/core/public'; -import { - sendRequest as sendStatelessRequest, - SendRequestConfig, - SendRequestResponse, -} from './send_request'; +import { sendRequest, SendRequestConfig } from './send_request'; export interface UseRequestConfig extends SendRequestConfig { pollIntervalMs?: number; @@ -37,7 +33,7 @@ export interface UseRequestResponse { isLoading: boolean; error: E | null; data?: D | null; - sendRequest: () => Promise>; + resendRequest: () => void; } export const useRequest = ( @@ -80,7 +76,7 @@ export const useRequest = ( /* eslint-disable-next-line react-hooks/exhaustive-deps */ }, [path, method, queryStringified, bodyStringified]); - const sendRequest = useCallback(async () => { + const resendRequest = useCallback(async () => { // If we're on an interval, this allows us to reset it if the user has manually requested the // data, to avoid doubled-up requests. clearPollInterval(); @@ -91,7 +87,7 @@ export const useRequest = ( // "old" error/data or loading state when a new request is in-flight. setIsLoading(true); - const response = await sendStatelessRequest(httpClient, requestBody); + const response = await sendRequest(httpClient, requestBody); const { data: serializedResponseData, error: responseError } = response; const isOutdatedRequest = requestId !== requestCountRef.current; @@ -99,7 +95,7 @@ export const useRequest = ( // Ignore outdated or irrelevant data. if (isOutdatedRequest || isUnmounted) { - return { data: null, error: null }; + return; } setError(responseError); @@ -112,8 +108,6 @@ export const useRequest = ( } // Setting isLoading to false also acts as a signal for scheduling the next poll request. setIsLoading(false); - - return { data: serializedResponseData, error: responseError }; }, [requestBody, httpClient, deserializer, clearPollInterval]); const scheduleRequest = useCallback(() => { @@ -121,19 +115,19 @@ export const useRequest = ( clearPollInterval(); if (pollIntervalMs) { - pollIntervalIdRef.current = setTimeout(sendRequest, pollIntervalMs); + pollIntervalIdRef.current = setTimeout(resendRequest, pollIntervalMs); } - }, [pollIntervalMs, sendRequest, clearPollInterval]); + }, [pollIntervalMs, resendRequest, clearPollInterval]); - // Send the request on component mount and whenever the dependencies of sendRequest() change. + // Send the request on component mount and whenever the dependencies of resendRequest() change. useEffect(() => { - sendRequest(); - }, [sendRequest]); + resendRequest(); + }, [resendRequest]); // Schedule the next poll request when the previous one completes. useEffect(() => { // When a request completes, attempt to schedule the next one. Note that we aren't re-scheduling - // a request whenever sendRequest's dependencies change. isLoading isn't set to false until the + // a request whenever resendRequest's dependencies change. isLoading isn't set to false until the // initial request has completed, so we won't schedule a request on mount. if (!isLoading) { scheduleRequest(); @@ -156,6 +150,6 @@ export const useRequest = ( isLoading, error, data, - sendRequest, // Gives the user the ability to manually request data + resendRequest, // Gives the user the ability to manually request data }; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/node_allocation.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/node_allocation.tsx index 6f80afccbff5e..6a22d8716514c 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/node_allocation.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/node_allocation.tsx @@ -52,7 +52,7 @@ export const NodeAllocation = ({ phaseData, isShowingErrors, }: React.PropsWithChildren>) => { - const { isLoading, data: nodes, error, sendRequest } = useLoadNodes(); + const { isLoading, data: nodes, error, resendRequest } = useLoadNodes(); const [selectedNodeAttrsForDetails, setSelectedNodeAttrsForDetails] = useState( null @@ -84,7 +84,7 @@ export const NodeAllocation = ({

{message} ({statusCode})

- + = ({ close, selectedNodeAttrs }) => { - const { data, isLoading, error, sendRequest } = useLoadNodeDetails(selectedNodeAttrs); + const { data, isLoading, error, resendRequest } = useLoadNodeDetails(selectedNodeAttrs); let content; if (isLoading) { content = ; @@ -47,7 +47,7 @@ export const NodeAttrsDetails: React.FunctionComponent = ({ close, select

{message} ({statusCode})

- + = ({ onChange, getUrlForApp, }) => { - const { error, isLoading, data, sendRequest } = useLoadSnapshotPolicies(); + const { error, isLoading, data, resendRequest } = useLoadSnapshotPolicies(); const policies = data.map((name: string) => ({ label: name, @@ -75,7 +75,7 @@ export const SnapshotPolicies: React.FunctionComponent = ({ { - const { error, isLoading, data: policies, sendRequest } = useLoadPoliciesList(false); + const { error, isLoading, data: policies, resendRequest } = useLoadPoliciesList(false); if (isLoading) { return ( } actions={ - + = navigateToApp, history, }) => { - const { data: policies, isLoading, error, sendRequest } = useLoadPoliciesList(true); + const { data: policies, isLoading, error, resendRequest } = useLoadPoliciesList(true); if (isLoading) { return ( @@ -53,7 +53,7 @@ export const PolicyTable: React.FunctionComponent =

} actions={ - + = policies={policies || []} history={history} navigateToApp={navigateToApp} - updatePolicies={sendRequest} + updatePolicies={resendRequest} /> ); }; diff --git a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/component_template_list.tsx b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/component_template_list.tsx index 8ba7409a9ac57..05f7f53969ded 100644 --- a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/component_template_list.tsx +++ b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/component_template_list.tsx @@ -42,7 +42,7 @@ export const ComponentTemplateList: React.FunctionComponent = ({ } = useGlobalFlyout(); const { api, trackMetric, documentation } = useComponentTemplatesContext(); - const { data, isLoading, error, sendRequest } = api.useLoadComponentTemplates(); + const { data, isLoading, error, resendRequest } = api.useLoadComponentTemplates(); const [componentTemplatesToDelete, setComponentTemplatesToDelete] = useState([]); @@ -170,7 +170,7 @@ export const ComponentTemplateList: React.FunctionComponent = ({ = ({ } else if (data && data.length === 0) { content = ; } else if (error) { - content = ; + content = ; } return ( @@ -194,7 +194,7 @@ export const ComponentTemplateList: React.FunctionComponent = ({ callback={(deleteResponse) => { if (deleteResponse?.hasDeletedComponentTemplates) { // refetch the component templates - sendRequest(); + resendRequest(); // go back to list view (if deleted from details flyout) goToComponentTemplateList(); } diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx index d37576f18e849..4f2a5c4a27b7a 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx @@ -49,7 +49,7 @@ export const DataStreamList: React.FunctionComponent {}; + reload: UseRequestResponse['resendRequest']; history: ScopedHistory; includeStats: boolean; filters?: string; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/legacy_templates/template_table/template_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/legacy_templates/template_table/template_table.tsx index 9203e76fce787..7ec6f1f94a2ab 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/legacy_templates/template_table/template_table.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/legacy_templates/template_table/template_table.tsx @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiInMemoryTable, EuiButton, EuiLink, EuiBasicTableColumn } from '@elastic/eui'; import { ScopedHistory } from 'kibana/public'; -import { SendRequestResponse, reactRouterNavigate } from '../../../../../../shared_imports'; +import { UseRequestResponse, reactRouterNavigate } from '../../../../../../shared_imports'; import { TemplateListItem } from '../../../../../../../common'; import { UIM_TEMPLATE_SHOW_DETAILS_CLICK } from '../../../../../../../common/constants'; import { TemplateDeleteModal } from '../../../../../components'; @@ -20,7 +20,7 @@ import { TemplateTypeIndicator } from '../../components'; interface Props { templates: TemplateListItem[]; - reload: () => Promise; + reload: UseRequestResponse['resendRequest']; editTemplate: (name: string, isLegacy?: boolean) => void; cloneTemplate: (name: string, isLegacy?: boolean) => void; history: ScopedHistory; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx index 5bacffc4c2404..94891297c857e 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx @@ -31,7 +31,7 @@ import { UIM_TEMPLATE_DETAIL_PANEL_ALIASES_TAB, UIM_TEMPLATE_DETAIL_PANEL_PREVIEW_TAB, } from '../../../../../../common/constants'; -import { SendRequestResponse } from '../../../../../shared_imports'; +import { UseRequestResponse } from '../../../../../shared_imports'; import { TemplateDeleteModal, SectionLoading, SectionError, Error } from '../../../../components'; import { useLoadIndexTemplate } from '../../../../services/api'; import { decodePathFromReactRouter } from '../../../../services/routing'; @@ -92,7 +92,7 @@ export interface Props { onClose: () => void; editTemplate: (name: string, isLegacy?: boolean) => void; cloneTemplate: (name: string, isLegacy?: boolean) => void; - reload: () => Promise; + reload: UseRequestResponse['resendRequest']; } export const TemplateDetailsContent = ({ diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx index f421bc5d87a54..c711f457123fb 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx @@ -59,7 +59,7 @@ export const TemplateList: React.FunctionComponent { const { uiMetricService } = useServices(); - const { error, isLoading, data: allTemplates, sendRequest: reload } = useLoadIndexTemplates(); + const { error, isLoading, data: allTemplates, resendRequest: reload } = useLoadIndexTemplates(); const [filters, setFilters] = useState>({ managed: { diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_table/template_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_table/template_table.tsx index 3dffdcde160f1..c32fd29cf9f92 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_table/template_table.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_table/template_table.tsx @@ -12,7 +12,7 @@ import { ScopedHistory } from 'kibana/public'; import { TemplateListItem } from '../../../../../../common'; import { UIM_TEMPLATE_SHOW_DETAILS_CLICK } from '../../../../../../common/constants'; -import { SendRequestResponse, reactRouterNavigate } from '../../../../../shared_imports'; +import { UseRequestResponse, reactRouterNavigate } from '../../../../../shared_imports'; import { encodePathForReactRouter } from '../../../../services/routing'; import { useServices } from '../../../../app_context'; import { TemplateDeleteModal } from '../../../../components'; @@ -21,7 +21,7 @@ import { TemplateTypeIndicator } from '../components'; interface Props { templates: TemplateListItem[]; - reload: () => Promise; + reload: UseRequestResponse['resendRequest']; editTemplate: (name: string) => void; cloneTemplate: (name: string) => void; history: ScopedHistory; diff --git a/x-pack/plugins/index_management/public/shared_imports.ts b/x-pack/plugins/index_management/public/shared_imports.ts index f7f992a090501..d58545768732e 100644 --- a/x-pack/plugins/index_management/public/shared_imports.ts +++ b/x-pack/plugins/index_management/public/shared_imports.ts @@ -8,6 +8,7 @@ export { SendRequestConfig, SendRequestResponse, UseRequestConfig, + UseRequestResponse, sendRequest, useRequest, Forms, diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_select_agent_policy.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_select_agent_policy.tsx index 9f48be54f866d..ccf9e45ebc4fa 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_select_agent_policy.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_select_agent_policy.tsx @@ -83,7 +83,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{ data: agentPoliciesData, error: agentPoliciesError, isLoading: isAgentPoliciesLoading, - sendRequest: refreshAgentPolicies, + resendRequest: refreshAgentPolicies, } = useGetAgentPolicies({ page: 1, perPage: 1000, diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/hooks/use_agent_status.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/hooks/use_agent_status.tsx index 71dcd728d5d1b..3483d8dee045a 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/hooks/use_agent_status.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/hooks/use_agent_status.tsx @@ -25,7 +25,7 @@ export function useGetAgentStatus(policyId?: string, options?: RequestOptions) { isLoading: agentStatusRequest.isLoading, data: agentStatusRequest.data, error: agentStatusRequest.error, - refreshAgentStatus: () => agentStatusRequest.sendRequest, + refreshAgentStatus: () => agentStatusRequest.resendRequest, }; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/list_page/index.tsx index 361b1c33f1a04..fb963dc67ae1c 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/list_page/index.tsx @@ -108,7 +108,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { ); // Fetch agent policies - const { isLoading, data: agentPolicyData, sendRequest } = useGetAgentPolicies({ + const { isLoading, data: agentPolicyData, resendRequest } = useGetAgentPolicies({ page: pagination.currentPage, perPage: pagination.pageSize, sortField: sorting?.field, @@ -204,7 +204,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { render: (agentPolicy: AgentPolicy) => ( sendRequest()} + onCopySuccess={() => resendRequest()} /> ), }, @@ -218,7 +218,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { } return cols; - }, [getHref, isFleetEnabled, sendRequest]); + }, [getHref, isFleetEnabled, resendRequest]); const createAgentPolicyButton = useMemo( () => ( @@ -270,7 +270,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { { setIsCreateAgentPolicyFlyoutOpen(false); - sendRequest(); + resendRequest(); }} /> ) : null} @@ -289,7 +289,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => { /> - sendRequest()}> + resendRequest()}> = () => { const { pagination, pageSizeOptions } = usePagination(); // Fetch data streams - const { isLoading, data: dataStreamsData, sendRequest } = useGetDataStreams(); + const { isLoading, data: dataStreamsData, resendRequest } = useGetDataStreams(); // Some policies retrieved, set up table props const columns = useMemo(() => { @@ -241,7 +241,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { key="reloadButton" color="primary" iconType="refresh" - onClick={() => sendRequest()} + onClick={() => resendRequest()} > = ({ ag [key: string]: JSX.Element; }>({}); - const { isLoading, data, sendRequest } = useGetOneAgentEvents(agent.id, { + const { isLoading, data, resendRequest } = useGetOneAgentEvents(agent.id, { page: pagination.currentPage, perPage: pagination.pageSize, kuery: search && search.trim() !== '' ? search.trim() : undefined, }); - const refresh = () => sendRequest(); + const refresh = () => resendRequest(); const total = data ? data.total : 0; const list = data ? data.list : []; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/index.tsx index 219b343eba41b..fe0781f4a240b 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_details_page/index.tsx @@ -51,7 +51,7 @@ export const AgentDetailsPage: React.FunctionComponent = () => { isInitialRequest, error, data: agentData, - sendRequest: sendAgentRequest, + resendRequest: sendAgentRequest, } = useGetOneAgent(agentId, { pollIntervalMs: 5000, }); diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx index 9548340df5b30..46f7ffb85b21f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx @@ -344,7 +344,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => { return ( agentsRequest.sendRequest()} + refresh={() => agentsRequest.resendRequest()} onReassignClick={() => setAgentToReassignId(agent.id)} /> ); @@ -394,7 +394,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => { agent={agentToReassign} onClose={() => { setAgentToReassignId(undefined); - agentsRequest.sendRequest(); + agentsRequest.resendRequest(); }} /> diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/enrollment_token_list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/enrollment_token_list_page/index.tsx index b3a4938b22310..d85a6e8b5b833 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/enrollment_token_list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/enrollment_token_list_page/index.tsx @@ -244,7 +244,10 @@ export const EnrollmentTokenListPage: React.FunctionComponent<{}> = () => { render: (_: any, apiKey: EnrollmentAPIKey) => { return ( apiKey.active && ( - enrollmentAPIKeysRequest.sendRequest()} /> + enrollmentAPIKeysRequest.resendRequest()} + /> ) ); }, @@ -258,7 +261,7 @@ export const EnrollmentTokenListPage: React.FunctionComponent<{}> = () => { agentPolicies={agentPolicies} onClose={() => { setFlyoutOpen(false); - enrollmentAPIKeysRequest.sendRequest(); + enrollmentAPIKeysRequest.resendRequest(); }} /> )} diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx index ccb50376dddb7..88148f1bc5746 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx @@ -51,7 +51,7 @@ export const PipelinesList: React.FunctionComponent = ({ const [pipelinesToDelete, setPipelinesToDelete] = useState([]); - const { data, isLoading, error, sendRequest } = services.api.useLoadPipelines(); + const { data, isLoading, error, resendRequest } = services.api.useLoadPipelines(); // Track component loaded useEffect(() => { @@ -98,7 +98,7 @@ export const PipelinesList: React.FunctionComponent = ({ } else if (data?.length) { content = ( = ({ defaultMessage="Unable to load pipelines. {reloadLink}" values={{ reloadLink: ( - + = ({ callback={(deleteResponse) => { if (deleteResponse?.hasDeletedPipelines) { // reload pipelines list - sendRequest(); + resendRequest(); setSelectedPipeline(undefined); goHome(); } diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_logistics.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_logistics.tsx index f825c7b1f3d98..7d3ba92cf2ad7 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_logistics.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_logistics.tsx @@ -51,7 +51,7 @@ export const PolicyStepLogistics: React.FunctionComponent = ({ name: undefined, }, }, - sendRequest: reloadRepositories, + resendRequest: reloadRepositories, } = useLoadRepositories(); const { i18n, history } = useServices(); diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/policy_details.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/policy_details.tsx index f67e8eb586238..b4612c9df42ff 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/policy_details.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/policy_details.tsx @@ -65,7 +65,7 @@ export const PolicyDetails: React.FunctionComponent = ({ onPolicyExecuted, }) => { const { i18n, uiMetricService, history } = useServices(); - const { error, data: policyDetails, sendRequest: reload } = useLoadPolicy(policyName); + const { error, data: policyDetails, resendRequest: reload } = useLoadPolicy(policyName); const [activeTab, setActiveTab] = useState(TAB_SUMMARY); const [isPopoverOpen, setIsPopoverOpen] = useState(false); diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx index 655bd0e9d8bb9..57f18ccbf8150 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_list.tsx @@ -45,7 +45,7 @@ export const PolicyList: React.FunctionComponent { diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx index d55bbf0b324cf..e7e4a9b54ada7 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx @@ -21,7 +21,7 @@ import { } from '@elastic/eui'; import { SlmPolicy } from '../../../../../../common/types'; -import { Error } from '../../../../../shared_imports'; +import { UseRequestResponse } from '../../../../../shared_imports'; import { UIM_POLICY_SHOW_DETAILS_CLICK } from '../../../../constants'; import { useServices } from '../../../../app_context'; import { @@ -30,13 +30,12 @@ import { PolicyDeleteProvider, } from '../../../../components'; import { linkToAddPolicy, linkToEditPolicy } from '../../../../services/navigation'; -import { SendRequestResponse } from '../../../../../shared_imports'; import { reactRouterNavigate } from '../../../../../../../../../src/plugins/kibana_react/public'; interface Props { policies: SlmPolicy[]; - reload: () => Promise>; + reload: UseRequestResponse['resendRequest']; openPolicyDetailsUrl: (name: SlmPolicy['name']) => string; onPolicyDeleted: (policiesDeleted: Array) => void; onPolicyExecuted: () => void; diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_list.tsx index 9afdad3806def..a3f57ce4fbf5e 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_list.tsx @@ -40,7 +40,7 @@ export const RepositoryList: React.FunctionComponent Promise>; + reload: UseRequestResponse['resendRequest']; openRepositoryDetailsUrl: (name: Repository['name']) => string; onRepositoryDeleted: (repositoriesDeleted: Array) => void; } diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx index d7a82386926c1..d9507a101bbac 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/restore_list/restore_list.tsx @@ -52,9 +52,13 @@ export const RestoreList: React.FunctionComponent = () => { const [currentInterval, setCurrentInterval] = useState(INTERVAL_OPTIONS[1]); // Load restores - const { error, isLoading, data: restores = [], isInitialRequest, sendRequest } = useLoadRestores( - currentInterval - ); + const { + error, + isLoading, + data: restores = [], + isInitialRequest, + resendRequest, + } = useLoadRestores(currentInterval); const { uiMetricService, history } = useServices(); @@ -174,7 +178,7 @@ export const RestoreList: React.FunctionComponent = () => { key={interval} icon="empty" onClick={() => { - sendRequest(); + resendRequest(); setCurrentInterval(interval); setIsIntervalMenuOpen(false); }} diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_list.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_list.tsx index d13188fc44730..97def33ffe8f6 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_list.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_list.tsx @@ -44,7 +44,7 @@ export const SnapshotList: React.FunctionComponent Promise>; + reload: UseRequestResponse['resendRequest']; openSnapshotDetailsUrl: (repositoryName: string, snapshotId: string) => string; repositoryFilter?: string; policyFilter?: string; diff --git a/x-pack/plugins/snapshot_restore/public/shared_imports.ts b/x-pack/plugins/snapshot_restore/public/shared_imports.ts index cad8ce147bd25..bd1c0e0cd395b 100644 --- a/x-pack/plugins/snapshot_restore/public/shared_imports.ts +++ b/x-pack/plugins/snapshot_restore/public/shared_imports.ts @@ -14,6 +14,7 @@ export { sendRequest, SendRequestConfig, SendRequestResponse, + UseRequestResponse, useAuthorizationContext, useRequest, UseRequestConfig, diff --git a/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/watch_visualization.tsx b/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/watch_visualization.tsx index 2ff0f53d07e91..935f0209e73c2 100644 --- a/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/watch_visualization.tsx +++ b/x-pack/plugins/watcher/public/application/sections/watch_edit/components/threshold_watch_edit/watch_visualization.tsx @@ -126,7 +126,7 @@ export const WatchVisualization = () => { isLoading, data: watchVisualizationData, error, - sendRequest: reload, + resendRequest: reload, } = useGetWatchVisualizationData(watchWithoutActions, visualizeOptions); useEffect( From 52fba21e4be4feb66cda91adcc808e0e709f9768 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Thu, 10 Sep 2020 11:44:53 -0400 Subject: [PATCH 05/29] Introduce telemetry for security features (#74530) Co-authored-by: Joe Portner <5295965+jportner@users.noreply.github.com> --- .../security/common/licensing/index.mock.ts | 1 + .../common/licensing/license_service.test.ts | 9 + .../common/licensing/license_service.ts | 3 + x-pack/plugins/security/kibana.json | 2 +- .../elasticsearch_privileges.test.tsx.snap | 1 + .../plugins/security/public/plugin.test.tsx | 2 + x-pack/plugins/security/server/config.test.ts | 64 +++ x-pack/plugins/security/server/config.ts | 10 +- x-pack/plugins/security/server/plugin.test.ts | 1 + x-pack/plugins/security/server/plugin.ts | 7 +- .../security/server/usage_collector/index.ts | 7 + .../security_usage_collector.test.ts | 465 ++++++++++++++++++ .../security_usage_collector.ts | 116 +++++ .../schema/xpack_plugins.json | 22 + 14 files changed, 706 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/security/server/usage_collector/index.ts create mode 100644 x-pack/plugins/security/server/usage_collector/security_usage_collector.test.ts create mode 100644 x-pack/plugins/security/server/usage_collector/security_usage_collector.ts diff --git a/x-pack/plugins/security/common/licensing/index.mock.ts b/x-pack/plugins/security/common/licensing/index.mock.ts index 06a7057abb87c..87225f479ceed 100644 --- a/x-pack/plugins/security/common/licensing/index.mock.ts +++ b/x-pack/plugins/security/common/licensing/index.mock.ts @@ -9,6 +9,7 @@ import { SecurityLicense } from '.'; export const licenseMock = { create: (): jest.Mocked => ({ + isLicenseAvailable: jest.fn(), isEnabled: jest.fn().mockReturnValue(true), getFeatures: jest.fn(), features$: of(), diff --git a/x-pack/plugins/security/common/licensing/license_service.test.ts b/x-pack/plugins/security/common/licensing/license_service.test.ts index 564b71a2e0fac..94aad8d3ac539 100644 --- a/x-pack/plugins/security/common/licensing/license_service.test.ts +++ b/x-pack/plugins/security/common/licensing/license_service.test.ts @@ -13,6 +13,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(undefined as any), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(false); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: false, @@ -34,6 +35,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(rawLicenseMock), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(false); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: false, @@ -60,6 +62,7 @@ describe('license features', function () { const subscriptionHandler = jest.fn(); const subscription = serviceSetup.license.features$.subscribe(subscriptionHandler); try { + expect(serviceSetup.license.isLicenseAvailable()).toEqual(false); expect(subscriptionHandler).toHaveBeenCalledTimes(1); expect(subscriptionHandler.mock.calls[0]).toMatchInlineSnapshot(` Array [ @@ -80,6 +83,7 @@ describe('license features', function () { `); rawLicense$.next(licenseMock.createLicenseMock()); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(subscriptionHandler).toHaveBeenCalledTimes(2); expect(subscriptionHandler.mock.calls[1]).toMatchInlineSnapshot(` Array [ @@ -112,6 +116,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(mockRawLicense), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: true, @@ -136,6 +141,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(mockRawLicense), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: false, allowLogin: false, @@ -159,6 +165,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(mockRawLicense), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: true, @@ -182,6 +189,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(mockRawLicense), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: true, @@ -205,6 +213,7 @@ describe('license features', function () { const serviceSetup = new SecurityLicenseService().setup({ license$: of(mockRawLicense), }); + expect(serviceSetup.license.isLicenseAvailable()).toEqual(true); expect(serviceSetup.license.getFeatures()).toEqual({ showLogin: true, allowLogin: true, diff --git a/x-pack/plugins/security/common/licensing/license_service.ts b/x-pack/plugins/security/common/licensing/license_service.ts index 75c7670f28a67..09b6ae95c282c 100644 --- a/x-pack/plugins/security/common/licensing/license_service.ts +++ b/x-pack/plugins/security/common/licensing/license_service.ts @@ -10,6 +10,7 @@ import { ILicense } from '../../../licensing/common/types'; import { SecurityLicenseFeatures } from './license_features'; export interface SecurityLicense { + isLicenseAvailable(): boolean; isEnabled(): boolean; getFeatures(): SecurityLicenseFeatures; features$: Observable; @@ -31,6 +32,8 @@ export class SecurityLicenseService { return { license: Object.freeze({ + isLicenseAvailable: () => rawLicense?.isAvailable ?? false, + isEnabled: () => this.isSecurityEnabledFromRawLicense(rawLicense), getFeatures: () => this.calculateFeaturesFromRawLicense(rawLicense), diff --git a/x-pack/plugins/security/kibana.json b/x-pack/plugins/security/kibana.json index 6a09e9e55a01b..40d7e293eaf66 100644 --- a/x-pack/plugins/security/kibana.json +++ b/x-pack/plugins/security/kibana.json @@ -4,7 +4,7 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "security"], "requiredPlugins": ["data", "features", "licensing", "taskManager"], - "optionalPlugins": ["home", "management"], + "optionalPlugins": ["home", "management", "usageCollection"], "server": true, "ui": true, "requiredBundles": [ diff --git a/x-pack/plugins/security/public/management/roles/edit_role/privileges/es/__snapshots__/elasticsearch_privileges.test.tsx.snap b/x-pack/plugins/security/public/management/roles/edit_role/privileges/es/__snapshots__/elasticsearch_privileges.test.tsx.snap index 1c020685c246d..a2e46af19bf34 100644 --- a/x-pack/plugins/security/public/management/roles/edit_role/privileges/es/__snapshots__/elasticsearch_privileges.test.tsx.snap +++ b/x-pack/plugins/security/public/management/roles/edit_role/privileges/es/__snapshots__/elasticsearch_privileges.test.tsx.snap @@ -184,6 +184,7 @@ exports[`it renders without crashing 1`] = ` }, "getFeatures": [MockFunction], "isEnabled": [MockFunction], + "isLicenseAvailable": [MockFunction], } } onChange={[MockFunction]} diff --git a/x-pack/plugins/security/public/plugin.test.tsx b/x-pack/plugins/security/public/plugin.test.tsx index 8cec4fbc2f5a2..8fe7d2805e18e 100644 --- a/x-pack/plugins/security/public/plugin.test.tsx +++ b/x-pack/plugins/security/public/plugin.test.tsx @@ -41,6 +41,7 @@ describe('Security Plugin', () => { __legacyCompat: { logoutUrl: '/some-base-path/logout', tenant: '/some-base-path' }, authc: { getCurrentUser: expect.any(Function), areAPIKeysEnabled: expect.any(Function) }, license: { + isLicenseAvailable: expect.any(Function), isEnabled: expect.any(Function), getFeatures: expect.any(Function), features$: expect.any(Observable), @@ -67,6 +68,7 @@ describe('Security Plugin', () => { expect(setupManagementServiceMock).toHaveBeenCalledWith({ authc: { getCurrentUser: expect.any(Function), areAPIKeysEnabled: expect.any(Function) }, license: { + isLicenseAvailable: expect.any(Function), isEnabled: expect.any(Function), getFeatures: expect.any(Function), features$: expect.any(Observable), diff --git a/x-pack/plugins/security/server/config.test.ts b/x-pack/plugins/security/server/config.test.ts index 520081ae30d8d..093a7643fbf64 100644 --- a/x-pack/plugins/security/server/config.test.ts +++ b/x-pack/plugins/security/server/config.test.ts @@ -904,11 +904,13 @@ describe('createConfig()', () => { }, "sortedProviders": Array [ Object { + "hasAccessAgreement": false, "name": "saml", "order": 0, "type": "saml", }, Object { + "hasAccessAgreement": false, "name": "basic", "order": 1, "type": "basic", @@ -982,6 +984,63 @@ describe('createConfig()', () => { ).toBe(true); }); + it('indicates which providers have the access agreement enabled', () => { + expect( + createConfig( + ConfigSchema.validate({ + authc: { + providers: { + basic: { basic1: { order: 3 } }, + saml: { + saml1: { order: 2, realm: 'saml1', accessAgreement: { message: 'foo' } }, + saml2: { order: 1, realm: 'saml2' }, + }, + oidc: { + oidc1: { order: 0, realm: 'oidc1', accessAgreement: { message: 'foo' } }, + oidc2: { order: 4, realm: 'oidc2' }, + }, + }, + }, + }), + loggingSystemMock.create().get(), + { isTLSEnabled: true } + ).authc.sortedProviders + ).toMatchInlineSnapshot(` + Array [ + Object { + "hasAccessAgreement": true, + "name": "oidc1", + "order": 0, + "type": "oidc", + }, + Object { + "hasAccessAgreement": false, + "name": "saml2", + "order": 1, + "type": "saml", + }, + Object { + "hasAccessAgreement": true, + "name": "saml1", + "order": 2, + "type": "saml", + }, + Object { + "hasAccessAgreement": false, + "name": "basic1", + "order": 3, + "type": "basic", + }, + Object { + "hasAccessAgreement": false, + "name": "oidc2", + "order": 4, + "type": "oidc", + }, + ] + `); + }); + it('correctly sorts providers based on the `order`', () => { expect( createConfig( @@ -1000,26 +1059,31 @@ describe('createConfig()', () => { ).toMatchInlineSnapshot(` Array [ Object { + "hasAccessAgreement": false, "name": "oidc1", "order": 0, "type": "oidc", }, Object { + "hasAccessAgreement": false, "name": "saml2", "order": 1, "type": "saml", }, Object { + "hasAccessAgreement": false, "name": "saml1", "order": 2, "type": "saml", }, Object { + "hasAccessAgreement": false, "name": "basic1", "order": 3, "type": "basic", }, Object { + "hasAccessAgreement": false, "name": "oidc2", "order": 4, "type": "oidc", diff --git a/x-pack/plugins/security/server/config.ts b/x-pack/plugins/security/server/config.ts index dcfe4825fb035..9ccbdac5e09f4 100644 --- a/x-pack/plugins/security/server/config.ts +++ b/x-pack/plugins/security/server/config.ts @@ -255,13 +255,19 @@ export function createConfig( type: keyof ProvidersConfigType; name: string; order: number; + hasAccessAgreement: boolean; }> = []; for (const [type, providerGroup] of Object.entries(providers)) { - for (const [name, { enabled, order }] of Object.entries(providerGroup ?? {})) { + for (const [name, { enabled, order, accessAgreement }] of Object.entries(providerGroup ?? {})) { if (!enabled) { delete providerGroup![name]; } else { - sortedProviders.push({ type: type as any, name, order }); + sortedProviders.push({ + type: type as any, + name, + order, + hasAccessAgreement: !!accessAgreement?.message, + }); } } } diff --git a/x-pack/plugins/security/server/plugin.test.ts b/x-pack/plugins/security/server/plugin.test.ts index 8d13f81075714..9825e77b164c8 100644 --- a/x-pack/plugins/security/server/plugin.test.ts +++ b/x-pack/plugins/security/server/plugin.test.ts @@ -108,6 +108,7 @@ describe('Security Plugin', () => { }, "getFeatures": [Function], "isEnabled": [Function], + "isLicenseAvailable": [Function], }, "registerSpacesService": [Function], } diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index 7d94e03916fa1..1eb406dd2061b 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -7,6 +7,7 @@ import { combineLatest } from 'rxjs'; import { first, map } from 'rxjs/operators'; import { TypeOf } from '@kbn/config-schema'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { deepFreeze, CoreSetup, @@ -32,6 +33,7 @@ import { AuditService, SecurityAuditLogger, AuditServiceSetup } from './audit'; import { SecurityFeatureUsageService, SecurityFeatureUsageServiceStart } from './feature_usage'; import { ElasticsearchService } from './elasticsearch'; import { SessionManagementService } from './session_management'; +import { registerSecurityUsageCollector } from './usage_collector'; export type SpacesService = Pick< SpacesPluginSetup['spacesService'], @@ -74,6 +76,7 @@ export interface PluginSetupDependencies { features: FeaturesPluginSetup; licensing: LicensingPluginSetup; taskManager: TaskManagerSetupContract; + usageCollection?: UsageCollectionSetup; } export interface PluginStartDependencies { @@ -123,7 +126,7 @@ export class Plugin { public async setup( core: CoreSetup, - { features, licensing, taskManager }: PluginSetupDependencies + { features, licensing, taskManager, usageCollection }: PluginSetupDependencies ) { const [config, legacyConfig] = await combineLatest([ this.initializerContext.config.create>().pipe( @@ -151,6 +154,8 @@ export class Plugin { this.featureUsageService.setup({ featureUsage: licensing.featureUsage }); + registerSecurityUsageCollector({ usageCollection, config, license }); + const audit = this.auditService.setup({ license, config: config.audit }); const auditLogger = new SecurityAuditLogger(audit.getLogger()); diff --git a/x-pack/plugins/security/server/usage_collector/index.ts b/x-pack/plugins/security/server/usage_collector/index.ts new file mode 100644 index 0000000000000..dd405ebac4241 --- /dev/null +++ b/x-pack/plugins/security/server/usage_collector/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { registerSecurityUsageCollector } from './security_usage_collector'; diff --git a/x-pack/plugins/security/server/usage_collector/security_usage_collector.test.ts b/x-pack/plugins/security/server/usage_collector/security_usage_collector.test.ts new file mode 100644 index 0000000000000..6c3dcddcdb418 --- /dev/null +++ b/x-pack/plugins/security/server/usage_collector/security_usage_collector.test.ts @@ -0,0 +1,465 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createConfig, ConfigSchema } from '../config'; +import { loggingSystemMock } from 'src/core/server/mocks'; +import { TypeOf } from '@kbn/config-schema'; +import { usageCollectionPluginMock } from 'src/plugins/usage_collection/server/mocks'; +import { registerSecurityUsageCollector } from './security_usage_collector'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { licenseMock } from '../../common/licensing/index.mock'; +import { SecurityLicenseFeatures } from '../../common/licensing'; + +describe('Security UsageCollector', () => { + const createSecurityConfig = (config: TypeOf) => { + return createConfig(config, loggingSystemMock.createLogger(), { isTLSEnabled: true }); + }; + + const createSecurityLicense = ({ + allowAccessAgreement = true, + allowAuditLogging = true, + allowRbac = true, + isLicenseAvailable, + }: Partial & { isLicenseAvailable: boolean }) => { + const license = licenseMock.create(); + license.isLicenseAvailable.mockReturnValue(isLicenseAvailable); + license.getFeatures.mockReturnValue({ + allowAccessAgreement, + allowAuditLogging, + allowRbac, + } as SecurityLicenseFeatures); + return license; + }; + + const clusterClient = elasticsearchServiceMock.createLegacyClusterClient(); + + describe('initialization', () => { + it('handles an undefined usage collector', () => { + const config = createSecurityConfig(ConfigSchema.validate({})); + const usageCollection = undefined; + const license = createSecurityLicense({ allowRbac: false, isLicenseAvailable: false }); + registerSecurityUsageCollector({ usageCollection, config, license }); + }); + + it('registers itself and waits for the license to become available before reporting itself as ready', async () => { + const config = createSecurityConfig(ConfigSchema.validate({})); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ allowRbac: false, isLicenseAvailable: false }); + + registerSecurityUsageCollector({ usageCollection, config, license }); + + expect(usageCollection.getCollectorByType('security')?.isReady()).toBe(false); + + license.isLicenseAvailable.mockReturnValue(true); + license.getFeatures.mockReturnValue({ allowRbac: true } as SecurityLicenseFeatures); + + expect(usageCollection.getCollectorByType('security')?.isReady()).toBe(true); + }); + }); + + it('reports correctly for a default configuration', async () => { + const config = createSecurityConfig(ConfigSchema.validate({})); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + + it('reports correctly when security is disabled in Elasticsearch', async () => { + const config = createSecurityConfig(ConfigSchema.validate({})); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ allowRbac: false, isLicenseAvailable: true }); + + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 0, + enabledAuthProviders: [], + loginSelectorEnabled: false, + httpAuthSchemes: [], + }); + }); + + describe('auth providers', () => { + it('does not report disabled auth providers', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + providers: { + basic: { + basic: { + order: 0, + }, + disabledBasic: { + enabled: false, + order: 1, + }, + }, + saml: { + disabledSaml: { + enabled: false, + realm: 'foo', + order: 2, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + + it('reports the types and count of enabled auth providers', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + providers: { + basic: { + basic: { + order: 0, + enabled: false, + }, + }, + saml: { + saml1: { + realm: 'foo', + order: 1, + }, + saml2: { + realm: 'bar', + order: 2, + }, + }, + pki: { + pki1: { + enabled: true, + order: 3, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 3, + enabledAuthProviders: ['saml', 'pki'], + loginSelectorEnabled: true, + httpAuthSchemes: ['apikey'], + }); + }); + }); + + describe('access agreement', () => { + it('reports if the access agreement message is configured for any provider', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + providers: { + saml: { + saml1: { + realm: 'foo', + order: 1, + accessAgreement: { + message: 'foo message', + }, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: true, + authProviderCount: 1, + enabledAuthProviders: ['saml'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + it('does not report the access agreement if the license does not permit it', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + providers: { + saml: { + saml1: { + realm: 'foo', + order: 1, + accessAgreement: { + message: 'foo message', + }, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ + isLicenseAvailable: true, + allowAccessAgreement: false, + }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['saml'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + + it('does not report the access agreement for disabled providers', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + providers: { + saml: { + saml1: { + enabled: false, + realm: 'foo', + order: 1, + accessAgreement: { + message: 'foo message', + }, + }, + saml2: { + realm: 'foo', + order: 2, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['saml'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + }); + + describe('login selector', () => { + it('reports when the login selector is enabled', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + selector: { + enabled: true, + }, + providers: { + saml: { + saml1: { + realm: 'foo', + order: 1, + showInSelector: true, + }, + }, + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['saml'], + loginSelectorEnabled: true, + httpAuthSchemes: ['apikey'], + }); + }); + }); + + describe('audit logging', () => { + it('reports when audit logging is enabled', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + audit: { + enabled: true, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true, allowAuditLogging: true }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: true, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + + it('does not report audit logging when the license does not permit it', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + audit: { + enabled: true, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true, allowAuditLogging: false }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['apikey'], + }); + }); + }); + + describe('http auth schemes', () => { + it('reports customized http auth schemes', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + http: { + schemes: ['basic', 'Negotiate'], + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true, allowAuditLogging: false }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['basic', 'Negotiate'], + }); + }); + + it('does not report auth schemes that are not "well known"', async () => { + const config = createSecurityConfig( + ConfigSchema.validate({ + authc: { + http: { + schemes: ['basic', 'Negotiate', 'customScheme'], + }, + }, + }) + ); + const usageCollection = usageCollectionPluginMock.createSetupContract(); + const license = createSecurityLicense({ isLicenseAvailable: true, allowAuditLogging: false }); + registerSecurityUsageCollector({ usageCollection, config, license }); + + const usage = await usageCollection + .getCollectorByType('security') + ?.fetch(clusterClient.asScoped().callAsCurrentUser); + + expect(usage).toEqual({ + auditLoggingEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 1, + enabledAuthProviders: ['basic'], + loginSelectorEnabled: false, + httpAuthSchemes: ['basic', 'Negotiate'], + }); + }); + }); +}); diff --git a/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts new file mode 100644 index 0000000000000..11e58f7f95fc2 --- /dev/null +++ b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { ConfigType } from '../config'; +import { SecurityLicense } from '../../common/licensing'; + +interface Usage { + auditLoggingEnabled: boolean; + loginSelectorEnabled: boolean; + accessAgreementEnabled: boolean; + authProviderCount: number; + enabledAuthProviders: string[]; + httpAuthSchemes: string[]; +} + +interface Deps { + usageCollection?: UsageCollectionSetup; + config: ConfigType; + license: SecurityLicense; +} + +// List of auth schemes collected from https://www.iana.org/assignments/http-authschemes/http-authschemes.xhtml +const WELL_KNOWN_AUTH_SCHEMES = [ + 'basic', + 'bearer', + 'digest', + 'hoba', + 'mutual', + 'negotiate', + 'oauth', + 'scram-sha-1', + 'scram-sha-256', + 'vapid', + 'apikey', // not part of the spec, but used by the Elastic Stack for API Key authentication +]; + +export function registerSecurityUsageCollector({ usageCollection, config, license }: Deps): void { + // usageCollection is an optional dependency, so make sure to return if it is not registered. + if (!usageCollection) { + return; + } + + // create usage collector + const securityCollector = usageCollection.makeUsageCollector({ + type: 'security', + isReady: () => license.isLicenseAvailable(), + schema: { + auditLoggingEnabled: { + type: 'boolean', + }, + loginSelectorEnabled: { + type: 'boolean', + }, + accessAgreementEnabled: { + type: 'boolean', + }, + authProviderCount: { + type: 'number', + }, + enabledAuthProviders: { + type: 'keyword', + }, + httpAuthSchemes: { + type: 'keyword', + }, + }, + fetch: () => { + const { allowRbac, allowAccessAgreement, allowAuditLogging } = license.getFeatures(); + if (!allowRbac) { + return { + auditLoggingEnabled: false, + loginSelectorEnabled: false, + accessAgreementEnabled: false, + authProviderCount: 0, + enabledAuthProviders: [], + httpAuthSchemes: [], + }; + } + + const auditLoggingEnabled = allowAuditLogging && config.audit.enabled; + const loginSelectorEnabled = config.authc.selector.enabled; + const authProviderCount = config.authc.sortedProviders.length; + const enabledAuthProviders = [ + ...new Set( + config.authc.sortedProviders.reduce( + (acc, provider) => [...acc, provider.type], + [] as string[] + ) + ), + ]; + const accessAgreementEnabled = + allowAccessAgreement && + config.authc.sortedProviders.some((provider) => provider.hasAccessAgreement); + + const httpAuthSchemes = config.authc.http.schemes.filter((scheme) => + WELL_KNOWN_AUTH_SCHEMES.includes(scheme.toLowerCase()) + ); + + return { + auditLoggingEnabled, + loginSelectorEnabled, + accessAgreementEnabled, + authProviderCount, + enabledAuthProviders, + httpAuthSchemes, + }; + }, + }); + + // register usage collector + usageCollection.registerCollector(securityCollector); +} diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index a7330d3ebd552..904b14a7459ad 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -297,6 +297,28 @@ } } }, + "security": { + "properties": { + "auditLoggingEnabled": { + "type": "boolean" + }, + "loginSelectorEnabled": { + "type": "boolean" + }, + "accessAgreementEnabled": { + "type": "boolean" + }, + "authProviderCount": { + "type": "number" + }, + "enabledAuthProviders": { + "type": "keyword" + }, + "httpAuthSchemes": { + "type": "keyword" + } + } + }, "spaces": { "properties": { "usesFeatureControls": { From 0207f82e801e70641f9b1b820e0425187aae0b22 Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Thu, 10 Sep 2020 12:14:54 -0400 Subject: [PATCH 06/29] Prevent editing/creation of these in the alerts management UI (#77097) --- .../public/alerts/cpu_usage_alert/cpu_usage_alert.tsx | 2 +- .../monitoring/public/alerts/legacy_alert/legacy_alert.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring/public/alerts/cpu_usage_alert/cpu_usage_alert.tsx b/x-pack/plugins/monitoring/public/alerts/cpu_usage_alert/cpu_usage_alert.tsx index 56cba83813a63..c9f82eb521433 100644 --- a/x-pack/plugins/monitoring/public/alerts/cpu_usage_alert/cpu_usage_alert.tsx +++ b/x-pack/plugins/monitoring/public/alerts/cpu_usage_alert/cpu_usage_alert.tsx @@ -23,6 +23,6 @@ export function createCpuUsageAlertType(): AlertTypeModel { ), validate, defaultActionMessage: '{{context.internalFullMessage}}', - requiresAppContext: false, + requiresAppContext: true, }; } diff --git a/x-pack/plugins/monitoring/public/alerts/legacy_alert/legacy_alert.tsx b/x-pack/plugins/monitoring/public/alerts/legacy_alert/legacy_alert.tsx index 58b37e43085ff..f6223d41ab30e 100644 --- a/x-pack/plugins/monitoring/public/alerts/legacy_alert/legacy_alert.tsx +++ b/x-pack/plugins/monitoring/public/alerts/legacy_alert/legacy_alert.tsx @@ -33,7 +33,7 @@ export function createLegacyAlertTypes(): AlertTypeModel[] { ), defaultActionMessage: '{{context.internalFullMessage}}', validate: () => ({ errors: {} }), - requiresAppContext: false, + requiresAppContext: true, }; }); } From c85a1b296ea84d51b45092c082440b63f9379983 Mon Sep 17 00:00:00 2001 From: Davis Plumlee <56367316+dplumlee@users.noreply.github.com> Date: Thu, 10 Sep 2020 13:02:34 -0400 Subject: [PATCH 07/29] [Security Solution] Updates rules table tooling (#76719) --- .../cypress/tasks/alerts_detection_rules.ts | 4 +- .../rules/all_rules_tables/index.tsx | 11 +--- .../__snapshots__/index.test.tsx.snap | 4 +- .../detection_engine/rules/api.test.ts | 4 +- .../containers/detection_engine/rules/api.ts | 20 ++++-- .../detection_engine/rules/types.ts | 3 +- .../detection_engine/rules/all/columns.tsx | 45 +++++++++++-- .../detection_engine/rules/all/index.tsx | 16 +++-- .../tags_filter_popover.tsx | 64 +++++++++++++++---- .../detection_engine/rules/translations.ts | 32 +++++++--- 10 files changed, 149 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts index 79756621ef502..5ec5bb97250db 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts_detection_rules.ts @@ -80,9 +80,9 @@ export const selectNumberOfRules = (numberOfRules: number) => { }; export const sortByActivatedRules = () => { - cy.get(SORT_RULES_BTN).click({ force: true }); + cy.get(SORT_RULES_BTN).contains('Activated').click({ force: true }); waitForRulesToBeLoaded(); - cy.get(SORT_RULES_BTN).click({ force: true }); + cy.get(SORT_RULES_BTN).contains('Activated').click({ force: true }); waitForRulesToBeLoaded(); }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/all_rules_tables/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/all_rules_tables/index.tsx index 8fd3f648bc812..bfb23ff6af6a0 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/all_rules_tables/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/all_rules_tables/index.tsx @@ -20,7 +20,7 @@ import { RulesColumns, RuleStatusRowItemType, } from '../../../pages/detection_engine/rules/all/columns'; -import { Rule, Rules } from '../../../containers/detection_engine/rules/types'; +import { Rule, Rules, RulesSortingFields } from '../../../containers/detection_engine/rules/types'; import { AllRulesTabs } from '../../../pages/detection_engine/rules/all'; // EuiBasicTable give me a hardtime with adding the ref attributes so I went the easy way @@ -30,7 +30,7 @@ const MyEuiBasicTable = styled(EuiBasicTable as any)`` as any; export interface SortingType { sort: { - field: 'enabled'; + field: RulesSortingFields; direction: Direction; }; } @@ -48,12 +48,7 @@ interface AllRulesTablesProps { rules: Rules; rulesColumns: RulesColumns[]; rulesStatuses: RuleStatusRowItemType[]; - sorting: { - sort: { - field: 'enabled'; - direction: Direction; - }; - }; + sorting: SortingType; tableOnChangeCallback: ({ page, sort }: EuiBasicTableOnChange) => void; tableRef?: React.MutableRefObject; selectedTab: AllRulesTabs; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/rule_actions_overflow/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/detections/components/rules/rule_actions_overflow/__snapshots__/index.test.tsx.snap index 1ed55774f935f..4d21a983c9707 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/rule_actions_overflow/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/detections/components/rules/rule_actions_overflow/__snapshots__/index.test.tsx.snap @@ -40,7 +40,7 @@ exports[`RuleActionsOverflow snapshots renders correctly against snapshot 1`] = icon="copy" onClick={[Function]} > - Duplicate rule… + Duplicate rule , - Delete rule… + Delete rule , ] } diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts index cd1ded544cfe5..2a15cf7b95ceb 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.test.ts @@ -202,7 +202,7 @@ describe('Detections Rules API', () => { expect(fetchMock).toHaveBeenCalledWith('/api/detection_engine/rules/_find', { method: 'GET', query: { - filter: 'alert.attributes.tags: "hello" AND alert.attributes.tags: "world"', + filter: 'alert.attributes.tags: "hello" OR alert.attributes.tags: "world"', page: 1, per_page: 20, sort_field: 'enabled', @@ -297,7 +297,7 @@ describe('Detections Rules API', () => { method: 'GET', query: { filter: - 'alert.attributes.name: ruleName AND alert.attributes.tags: "__internal_immutable:false" AND alert.attributes.tags: "__internal_immutable:true" AND alert.attributes.tags: "hello" AND alert.attributes.tags: "world"', + 'alert.attributes.name: ruleName AND alert.attributes.tags: "__internal_immutable:false" AND alert.attributes.tags: "__internal_immutable:true" AND (alert.attributes.tags: "hello" OR alert.attributes.tags: "world")', page: 1, per_page: 20, sort_field: 'enabled', diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts index e254516d11076..b66154fbb57d2 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts @@ -107,7 +107,7 @@ export const fetchRules = async ({ }, signal, }: FetchRulesProps): Promise => { - const filters = [ + const filtersWithoutTags = [ ...(filterOptions.filter.length ? [`alert.attributes.name: ${filterOptions.filter}`] : []), ...(filterOptions.showCustomRules ? [`alert.attributes.tags: "__internal_immutable:false"`] @@ -115,15 +115,27 @@ export const fetchRules = async ({ ...(filterOptions.showElasticRules ? [`alert.attributes.tags: "__internal_immutable:true"`] : []), + ].join(' AND '); + + const tags = [ ...(filterOptions.tags?.map((t) => `alert.attributes.tags: "${t.replace(/"/g, '\\"')}"`) ?? []), - ]; + ].join(' OR '); + + const filterString = + filtersWithoutTags !== '' && tags !== '' + ? `${filtersWithoutTags} AND (${tags})` + : filtersWithoutTags + tags; + + const getFieldNameForSortField = (field: string) => { + return field === 'name' ? `${field}.keyword` : field; + }; const query = { page: pagination.page, per_page: pagination.perPage, - sort_field: filterOptions.sortField, + sort_field: getFieldNameForSortField(filterOptions.sortField), sort_order: filterOptions.sortOrder, - ...(filters.length ? { filter: filters.join(' AND ') } : {}), + ...(filterString !== '' ? { filter: filterString } : {}), }; return KibanaServices.get().http.fetch( diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts index e94e57ad82bcf..49579e893029b 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts @@ -149,9 +149,10 @@ export interface FetchRulesProps { signal: AbortSignal; } +export type RulesSortingFields = 'enabled' | 'updated_at' | 'name' | 'created_at'; export interface FilterOptions { filter: string; - sortField: string; + sortField: RulesSortingFields; sortOrder: SortOrder; showCustomRules?: boolean; showElasticRules?: boolean; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx index ea36a0cb0b48d..866d3e896a71d 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx @@ -99,7 +99,6 @@ interface GetColumns { reFetchRules: (refreshPrePackagedRule?: boolean) => void; } -// Michael: Are we able to do custom, in-table-header filters, as shown in my wireframes? export const getColumns = ({ dispatch, dispatchToaster, @@ -127,7 +126,8 @@ export const getColumns = ({ ), truncateText: true, - width: '24%', + width: '20%', + sortable: true, }, { field: 'risk_score', @@ -138,14 +138,14 @@ export const getColumns = ({ ), truncateText: true, - width: '14%', + width: '10%', }, { field: 'severity', name: i18n.COLUMN_SEVERITY, render: (value: Rule['severity']) => , truncateText: true, - width: '16%', + width: '12%', }, { field: 'status_date', @@ -160,7 +160,7 @@ export const getColumns = ({ ); }, truncateText: true, - width: '20%', + width: '14%', }, { field: 'status', @@ -174,9 +174,40 @@ export const getColumns = ({ ); }, - width: '16%', + width: '12%', truncateText: true, }, + { + field: 'updated_at', + name: i18n.COLUMN_LAST_UPDATE, + render: (value: Rule['updated_at']) => { + return value == null ? ( + getEmptyTagValue() + ) : ( + + + + ); + }, + sortable: true, + truncateText: true, + width: '14%', + }, + { + field: 'version', + name: i18n.COLUMN_VERSION, + render: (value: Rule['version']) => { + return value == null ? ( + getEmptyTagValue() + ) : ( + + {value} + + ); + }, + truncateText: true, + width: '10%', + }, { field: 'tags', name: i18n.COLUMN_TAGS, @@ -190,7 +221,7 @@ export const getColumns = ({ ), truncateText: true, - width: '20%', + width: '14%', }, { align: 'center', diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/index.tsx index 110691328b13b..306adbd63ee72 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/index.tsx @@ -24,6 +24,7 @@ import { Rule, PaginationOptions, exportRules, + RulesSortingFields, } from '../../../../containers/detection_engine/rules'; import { HeaderSection } from '../../../../../common/components/header_section'; import { @@ -53,12 +54,12 @@ import { hasMlLicense } from '../../../../../../common/machine_learning/has_ml_l import { SecurityPageName } from '../../../../../app/types'; import { useFormatUrl } from '../../../../../common/components/link_to'; -const SORT_FIELD = 'enabled'; +const INITIAL_SORT_FIELD = 'enabled'; const initialState: State = { exportRuleIds: [], filterOptions: { filter: '', - sortField: SORT_FIELD, + sortField: INITIAL_SORT_FIELD, sortOrder: 'desc', }, loadingRuleIds: [], @@ -164,8 +165,13 @@ export const AllRules = React.memo( }); const sorting = useMemo( - (): SortingType => ({ sort: { field: 'enabled', direction: filterOptions.sortOrder } }), - [filterOptions.sortOrder] + (): SortingType => ({ + sort: { + field: filterOptions.sortField, + direction: filterOptions.sortOrder, + }, + }), + [filterOptions] ); const prePackagedRuleStatus = getPrePackagedRuleStatus( @@ -215,7 +221,7 @@ export const AllRules = React.memo( dispatch({ type: 'updateFilterOptions', filterOptions: { - sortField: SORT_FIELD, // Only enabled is supported for sorting currently + sortField: (sort?.field as RulesSortingFields) ?? INITIAL_SORT_FIELD, // Narrowing EuiBasicTable sorting types sortOrder: sort?.direction ?? 'desc', }, pagination: { page: page.index + 1, perPage: page.size }, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx index 49fe3438664c6..4fe0bc8f835df 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx @@ -4,7 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { Dispatch, SetStateAction, useState } from 'react'; +import React, { + ChangeEvent, + Dispatch, + SetStateAction, + useCallback, + useEffect, + useMemo, + useState, +} from 'react'; import { EuiFilterButton, EuiFilterSelectItem, @@ -13,6 +21,8 @@ import { EuiPanel, EuiPopover, EuiText, + EuiFieldSearch, + EuiPopoverTitle, } from '@elastic/eui'; import styled from 'styled-components'; import * as i18n from '../../translations'; @@ -37,12 +47,39 @@ const ScrollableDiv = styled.div` * @param tags to display for filtering * @param onSelectedTagsChanged change listener to be notified when tag selection changes */ -export const TagsFilterPopoverComponent = ({ +const TagsFilterPopoverComponent = ({ tags, selectedTags, onSelectedTagsChanged, }: TagsFilterPopoverProps) => { + const sortedTags = useMemo(() => { + return tags.sort((a: string, b: string) => a.toLowerCase().localeCompare(b.toLowerCase())); // Case insensitive + }, [tags]); const [isTagPopoverOpen, setIsTagPopoverOpen] = useState(false); + const [searchInput, setSearchInput] = useState(''); + const [filterTags, setFilterTags] = useState(sortedTags); + + const tagsComponent = useMemo(() => { + return filterTags.map((tag, index) => ( + toggleSelectedGroup(tag, selectedTags, onSelectedTagsChanged)} + > + {`${tag}`} + + )); + }, [onSelectedTagsChanged, selectedTags, filterTags]); + + const onSearchInputChange = useCallback((event: ChangeEvent) => { + setSearchInput(event.target.value); + }, []); + + useEffect(() => { + setFilterTags( + sortedTags.filter((tag) => tag.toLowerCase().includes(searchInput.toLowerCase())) + ); + }, [sortedTags, searchInput]); return ( - - {tags.map((tag, index) => ( - toggleSelectedGroup(tag, selectedTags, onSelectedTagsChanged)} - > - {`${tag}`} - - ))} - - {tags.length === 0 && ( + + + + {tagsComponent} + {filterTags.length === 0 && ( diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts index b20c8de8ed58b..09503fcf1ef0f 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/translations.ts @@ -16,7 +16,7 @@ export const BACK_TO_DETECTIONS = i18n.translate( export const IMPORT_RULE = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.importRuleTitle', { - defaultMessage: 'Import rule…', + defaultMessage: 'Import rule', } ); @@ -100,7 +100,7 @@ export const BATCH_ACTION_ACTIVATE_SELECTED_ERROR = (totalRules: number) => 'xpack.securitySolution.detectionEngine.rules.allRules.batchActions.activateSelectedErrorTitle', { values: { totalRules }, - defaultMessage: 'Error activating {totalRules, plural, =1 {rule} other {rules}}…', + defaultMessage: 'Error activating {totalRules, plural, =1 {rule} other {rules}}', } ); @@ -116,7 +116,7 @@ export const BATCH_ACTION_DEACTIVATE_SELECTED_ERROR = (totalRules: number) => 'xpack.securitySolution.detectionEngine.rules.allRules.batchActions.deactivateSelectedErrorTitle', { values: { totalRules }, - defaultMessage: 'Error deactivating {totalRules, plural, =1 {rule} other {rules}}…', + defaultMessage: 'Error deactivating {totalRules, plural, =1 {rule} other {rules}}', } ); @@ -130,14 +130,14 @@ export const BATCH_ACTION_EXPORT_SELECTED = i18n.translate( export const BATCH_ACTION_DUPLICATE_SELECTED = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.batchActions.duplicateSelectedTitle', { - defaultMessage: 'Duplicate selected…', + defaultMessage: 'Duplicate selected', } ); export const BATCH_ACTION_DELETE_SELECTED = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.batchActions.deleteSelectedTitle', { - defaultMessage: 'Delete selected…', + defaultMessage: 'Delete selected', } ); @@ -153,7 +153,7 @@ export const BATCH_ACTION_DELETE_SELECTED_ERROR = (totalRules: number) => 'xpack.securitySolution.detectionEngine.rules.allRules.batchActions.deleteSelectedErrorTitle', { values: { totalRules }, - defaultMessage: 'Error deleting {totalRules, plural, =1 {rule} other {rules}}…', + defaultMessage: 'Error deleting {totalRules, plural, =1 {rule} other {rules}}', } ); @@ -224,7 +224,7 @@ export const DUPLICATE = i18n.translate( export const DUPLICATE_RULE = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.actions.duplicateRuleDescription', { - defaultMessage: 'Duplicate rule…', + defaultMessage: 'Duplicate rule', } ); @@ -241,7 +241,7 @@ export const SUCCESSFULLY_DUPLICATED_RULES = (totalRules: number) => export const DUPLICATE_RULE_ERROR = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.actions.duplicateRuleErrorDescription', { - defaultMessage: 'Error duplicating rule…', + defaultMessage: 'Error duplicating rule', } ); @@ -255,7 +255,7 @@ export const EXPORT_RULE = i18n.translate( export const DELETE_RULE = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.actions.deleteeRuleDescription', { - defaultMessage: 'Delete rule…', + defaultMessage: 'Delete rule', } ); @@ -287,6 +287,13 @@ export const COLUMN_LAST_COMPLETE_RUN = i18n.translate( } ); +export const COLUMN_LAST_UPDATE = i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.allRules.columns.lastUpdateTitle', + { + defaultMessage: 'Last updated', + } +); + export const COLUMN_LAST_RESPONSE = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.columns.lastResponseTitle', { @@ -294,6 +301,13 @@ export const COLUMN_LAST_RESPONSE = i18n.translate( } ); +export const COLUMN_VERSION = i18n.translate( + 'xpack.securitySolution.detectionEngine.rules.allRules.columns.versionTitle', + { + defaultMessage: 'Version', + } +); + export const COLUMN_TAGS = i18n.translate( 'xpack.securitySolution.detectionEngine.rules.allRules.columns.tagsTitle', { From a55edc99371b6928fe112dcb1a5a1821ff7e62be Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Thu, 10 Sep 2020 13:10:20 -0400 Subject: [PATCH 08/29] [ML] Adds Metadata and Discovery Analysis Jobs to Security Integration (#76023) * adds enhanced winlogbeat module * adds enhanced auditbeat module * splits discovery jobs * fixes winlogbeat manifest * adds process group * adds custom urls * adds by field as influencer * use process.title as influencer * updates custom url Co-authored-by: Elastic Machine --- .../modules/siem_auditbeat/manifest.json | 90 +++++++++++++++++++ ...linux_network_configuration_discovery.json | 26 ++++++ ...ed_linux_network_connection_discovery.json | 23 +++++ ...ed_linux_rare_kernel_module_arguments.json | 22 +++++ .../datafeed_linux_rare_metadata_process.json | 12 +++ .../ml/datafeed_linux_rare_metadata_user.json | 12 +++ .../ml/datafeed_linux_rare_sudo_user.json | 15 ++++ .../ml/datafeed_linux_rare_user_compiler.json | 22 +++++ ...ed_linux_system_information_discovery.json | 31 +++++++ ...tafeed_linux_system_process_discovery.json | 21 +++++ .../datafeed_linux_system_user_discovery.json | 23 +++++ ...linux_network_configuration_discovery.json | 53 +++++++++++ .../linux_network_connection_discovery.json | 53 +++++++++++ .../linux_rare_kernel_module_arguments.json | 45 ++++++++++ .../ml/linux_rare_metadata_process.json | 52 +++++++++++ .../ml/linux_rare_metadata_user.json | 43 +++++++++ .../ml/linux_rare_sudo_user.json | 53 +++++++++++ .../ml/linux_rare_user_compiler.json | 45 ++++++++++ .../linux_system_information_discovery.json | 53 +++++++++++ .../ml/linux_system_process_discovery.json | 53 +++++++++++ .../ml/linux_system_user_discovery.json | 53 +++++++++++ .../modules/siem_winlogbeat/manifest.json | 20 ++++- ...atafeed_windows_rare_metadata_process.json | 12 +++ .../datafeed_windows_rare_metadata_user.json | 12 +++ .../ml/windows_rare_metadata_process.json | 52 +++++++++++ .../ml/windows_rare_metadata_user.json | 43 +++++++++ 26 files changed, 938 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_configuration_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_connection_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_kernel_module_arguments.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_sudo_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_user_compiler.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_information_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_process_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_user_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_configuration_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_connection_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_kernel_module_arguments.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_sudo_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_user_compiler.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_information_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_process_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_user_discovery.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_user.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/manifest.json index 1e7fcdd4320f8..36d1df6db4c99 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/manifest.json @@ -40,6 +40,46 @@ { "id": "linux_anomalous_user_name_ecs", "file": "linux_anomalous_user_name_ecs.json" + }, + { + "id": "linux_rare_metadata_process", + "file": "linux_rare_metadata_process.json" + }, + { + "id": "linux_rare_metadata_user", + "file": "linux_rare_metadata_user.json" + }, + { + "id": "linux_rare_user_compiler", + "file": "linux_rare_user_compiler.json" + }, + { + "id": "linux_rare_kernel_module_arguments", + "file": "linux_rare_kernel_module_arguments.json" + }, + { + "id": "linux_rare_sudo_user", + "file": "linux_rare_sudo_user.json" + }, + { + "id": "linux_system_user_discovery", + "file": "linux_system_user_discovery.json" + }, + { + "id": "linux_system_information_discovery", + "file": "linux_system_information_discovery.json" + }, + { + "id": "linux_system_process_discovery", + "file": "linux_system_process_discovery.json" + }, + { + "id": "linux_network_connection_discovery", + "file": "linux_network_connection_discovery.json" + }, + { + "id": "linux_network_configuration_discovery", + "file": "linux_network_configuration_discovery.json" } ], "datafeeds": [ @@ -77,6 +117,56 @@ "id": "datafeed-linux_anomalous_user_name_ecs", "file": "datafeed_linux_anomalous_user_name_ecs.json", "job_id": "linux_anomalous_user_name_ecs" + }, + { + "id": "datafeed-linux_rare_metadata_process", + "file": "datafeed_linux_rare_metadata_process.json", + "job_id": "linux_rare_metadata_process" + }, + { + "id": "datafeed-linux_rare_metadata_user", + "file": "datafeed_linux_rare_metadata_user.json", + "job_id": "linux_rare_metadata_user" + }, + { + "id": "datafeed-linux_rare_user_compiler", + "file": "datafeed_linux_rare_user_compiler.json", + "job_id": "linux_rare_user_compiler" + }, + { + "id": "datafeed-linux_rare_kernel_module_arguments", + "file": "datafeed_linux_rare_kernel_module_arguments.json", + "job_id": "linux_rare_kernel_module_arguments" + }, + { + "id": "datafeed-linux_rare_sudo_user", + "file": "datafeed_linux_rare_sudo_user.json", + "job_id": "linux_rare_sudo_user" + }, + { + "id": "datafeed-linux_system_information_discovery", + "file": "datafeed_linux_system_information_discovery.json", + "job_id": "linux_system_information_discovery" + }, + { + "id": "datafeed-linux_system_process_discovery", + "file": "datafeed_linux_system_process_discovery.json", + "job_id": "linux_system_process_discovery" + }, + { + "id": "datafeed-linux_system_user_discovery", + "file": "datafeed_linux_system_user_discovery.json", + "job_id": "linux_system_user_discovery" + }, + { + "id": "datafeed-linux_network_configuration_discovery", + "file": "datafeed_linux_network_configuration_discovery.json", + "job_id": "linux_network_configuration_discovery" + }, + { + "id": "datafeed-linux_network_connection_discovery", + "file": "datafeed_linux_network_connection_discovery.json", + "job_id": "linux_network_connection_discovery" } ] } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_configuration_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_configuration_discovery.json new file mode 100644 index 0000000000000..d4a130770c920 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_configuration_discovery.json @@ -0,0 +1,26 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + {"term": {"process.name": "arp"}}, + {"term": {"process.name": "echo"}}, + {"term": {"process.name": "ethtool"}}, + {"term": {"process.name": "ifconfig"}}, + {"term": {"process.name": "ip"}}, + {"term": {"process.name": "iptables"}}, + {"term": {"process.name": "ufw"}} + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_connection_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_connection_discovery.json new file mode 100644 index 0000000000000..0ae80df4bd47d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_connection_discovery.json @@ -0,0 +1,23 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + {"term": {"process.name": "netstat"}}, + {"term": {"process.name": "ss"}}, + {"term": {"process.name": "route"}}, + {"term": {"process.name": "showmount"}} + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_kernel_module_arguments.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_kernel_module_arguments.json new file mode 100644 index 0000000000000..99bb690c8d73d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_kernel_module_arguments.json @@ -0,0 +1,22 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"exists": {"field": "process.title"}}], + "must": [ + {"bool": { + "should": [ + {"term": {"process.name": "insmod"}}, + {"term": {"process.name": "kmod"}}, + {"term": {"process.name": "modprobe"}}, + {"term": {"process.name": "rmod"}} + ] + }} + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_process.json new file mode 100644 index 0000000000000..dc0f6c4e81b33 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_process.json @@ -0,0 +1,12 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"term": {"destination.ip": "169.254.169.254"}}] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_user.json new file mode 100644 index 0000000000000..dc0f6c4e81b33 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_user.json @@ -0,0 +1,12 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"term": {"destination.ip": "169.254.169.254"}}] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_sudo_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_sudo_user.json new file mode 100644 index 0000000000000..544675f3d48dc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_sudo_user.json @@ -0,0 +1,15 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + {"term": {"event.action": "executed"}}, + {"term": {"process.name": "sudo"}} + ] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_user_compiler.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_user_compiler.json new file mode 100644 index 0000000000000..027b124010001 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_user_compiler.json @@ -0,0 +1,22 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"term": {"event.action": "executed"}}], + "must": [ + {"bool": { + "should": [ + {"term": {"process.name": "compile"}}, + {"term": {"process.name": "gcc"}}, + {"term": {"process.name": "make"}}, + {"term": {"process.name": "yasm"}} + ] + }} + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_information_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_information_discovery.json new file mode 100644 index 0000000000000..6e7ce26763f79 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_information_discovery.json @@ -0,0 +1,31 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + {"term": {"process.name": "cat"}}, + {"term": {"process.name": "grep"}}, + {"term": {"process.name": "head"}}, + {"term": {"process.name": "hostname"}}, + {"term": {"process.name": "less"}}, + {"term": {"process.name": "ls"}}, + {"term": {"process.name": "lsmod"}}, + {"term": {"process.name": "more"}}, + {"term": {"process.name": "strings"}}, + {"term": {"process.name": "tail"}}, + {"term": {"process.name": "uptime"}}, + {"term": {"process.name": "uname"}} + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_process_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_process_discovery.json new file mode 100644 index 0000000000000..dbd8f54ff9712 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_process_discovery.json @@ -0,0 +1,21 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + {"term": {"process.name": "ps"}}, + {"term": {"process.name": "top"}} + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_user_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_user_discovery.json new file mode 100644 index 0000000000000..24230094a47d2 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_user_discovery.json @@ -0,0 +1,23 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + {"term": {"process.name": "users"}}, + {"term": {"process.name": "w"}}, + {"term": {"process.name": "who"}}, + {"term": {"process.name": "whoami"}} + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_configuration_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_configuration_discovery.json new file mode 100644 index 0000000000000..6d687764085e0 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_configuration_discovery.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for commands related to system network configuration discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used by a threat actor to engage in system network configuration discovery in order to increase their understanding of connected networks and hosts. This information may be used to shape follow-up behaviors such as lateral movement or additional discovery.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_connection_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_connection_discovery.json new file mode 100644 index 0000000000000..b41439548dd59 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_connection_discovery.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for commands related to system network connection discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used by a threat actor to engage in system network connection discovery in order to increase their understanding of connected services and systems. This information may be used to shape follow-up behaviors such as lateral movement or additional discovery.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_kernel_module_arguments.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_kernel_module_arguments.json new file mode 100644 index 0000000000000..1b79e83054251 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_kernel_module_arguments.json @@ -0,0 +1,45 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for unusual kernel modules which are often used for stealth.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.title\"", + "function": "rare", + "by_field_name": "process.title" + } + ], + "influencers": [ + "process.title", + "process.working_directory", + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_process.json new file mode 100644 index 0000000000000..7295f11e600d7 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_process.json @@ -0,0 +1,52 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for anomalous access to the metadata service by an unusual process. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "host.name", + "user.name", + "process.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_user.json new file mode 100644 index 0000000000000..049d10920de00 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_metadata_user.json @@ -0,0 +1,43 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for anomalous access to the metadata service by an unusual user. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_sudo_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_sudo_user.json new file mode 100644 index 0000000000000..654f5c76e5698 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_sudo_user.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for sudo activity from an unusual user context.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_user_compiler.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_user_compiler.json new file mode 100644 index 0000000000000..245b7e0819c7d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_rare_user_compiler.json @@ -0,0 +1,45 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for compiler activity by a user context which does not normally run compilers. This can be ad-hoc software changes or unauthorized software deployment. This can also be due to local privliege elevation via locally run exploits or malware activity.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.title", + "host.name", + "process.working_directory", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_information_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_information_discovery.json new file mode 100644 index 0000000000000..3a51223b4899c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_information_discovery.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for commands related to system information discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used to engage in system information discovery in order to gather detailed information about system configuration and software versions. This may be a precursor to selection of a persistence mechanism or a method of privilege elevation.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "16mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_process_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_process_discovery.json new file mode 100644 index 0000000000000..592bb5a717fc0 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_process_discovery.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for commands related to system process discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used to engage in system process discovery in order to increase their understanding of software applications running on a target host or network. This may be a precursor to selection of a persistence mechanism or a method of privilege elevation.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "16mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_user_discovery.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_user_discovery.json new file mode 100644 index 0000000000000..33f42c274b337 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_system_user_discovery.json @@ -0,0 +1,53 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Auditbeat - Looks for commands related to system user or owner discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used to engage in system owner or user discovery in order to identify currently active or primary users of a system. This may be a precursor to additional discovery, credential dumping or privilege elevation activity.", + "groups": [ + "security", + "auditbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "process.args", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "16mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-auditbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/manifest.json index ffbf5aa7d8bb0..969873ead6d9c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/manifest.json @@ -48,6 +48,14 @@ { "id": "windows_rare_user_runas_event", "file": "windows_rare_user_runas_event.json" + }, + { + "id": "windows_rare_metadata_process", + "file": "windows_rare_metadata_process.json" + }, + { + "id": "windows_rare_metadata_user", + "file": "windows_rare_metadata_user.json" } ], "datafeeds": [ @@ -95,6 +103,16 @@ "id": "datafeed-windows_rare_user_runas_event", "file": "datafeed_windows_rare_user_runas_event.json", "job_id": "windows_rare_user_runas_event" + }, + { + "id": "datafeed-windows_rare_metadata_process", + "file": "datafeed_windows_rare_metadata_process.json", + "job_id": "windows_rare_metadata_process" + }, + { + "id": "datafeed-windows_rare_metadata_user", + "file": "datafeed_windows_rare_metadata_user.json", + "job_id": "windows_rare_metadata_user" } ] -} +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_process.json new file mode 100644 index 0000000000000..dc0f6c4e81b33 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_process.json @@ -0,0 +1,12 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"term": {"destination.ip": "169.254.169.254"}}] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_user.json new file mode 100644 index 0000000000000..dc0f6c4e81b33 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/datafeed_windows_rare_metadata_user.json @@ -0,0 +1,12 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [{"term": {"destination.ip": "169.254.169.254"}}] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_process.json new file mode 100644 index 0000000000000..85fddbcc53e0f --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_process.json @@ -0,0 +1,52 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Winlogbeat - Looks for anomalous access to the metadata service by an unusual process. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "winlogbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-winlogbeat", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_user.json new file mode 100644 index 0000000000000..767c2d5b30ad2 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat/ml/windows_rare_metadata_user.json @@ -0,0 +1,43 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Winlogbeat - Looks for anomalous access to the metadata service by an unusual user. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "winlogbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-siem-winlogbeat", + "custom_urls": [ + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } + } From f8bb47880030edba7ff943d72ce05c86b21cbb9a Mon Sep 17 00:00:00 2001 From: John Schulz Date: Thu, 10 Sep 2020 13:18:57 -0400 Subject: [PATCH 09/29] [Ingest Manager] getInstallType type improvements (#77053) * Add overloads to getInstallType. Remove 2 ignores. * Move tests inside `it` blocks * Add ts-expect-error for InstallType invariants --- .../server/routes/epm/handlers.ts | 2 +- .../services/epm/packages/install.test.ts | 72 +++++++++++++------ .../server/services/epm/packages/install.ts | 41 +++++++---- 3 files changed, 77 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index b19960cc90228..385e256933c12 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -167,7 +167,7 @@ export const installPackageHandler: RequestHandler< await removeInstallation({ savedObjectsClient, pkgkey, callCluster }); } if (installType === 'update') { - // @ts-ignore installType conditions already check for existence of installedPkg + // @ts-ignore getInstallType ensures we have installedPkg const prevVersion = `${pkgName}-${installedPkg.attributes.version}`; logger.error(`rolling back to ${prevVersion} after error installing ${pkgkey}`); await installPackage({ diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts index cc26e631a6215..2f60c74d3514f 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts @@ -42,36 +42,62 @@ const mockInstallationUpdateFail: SavedObject = { }; describe('install', () => { describe('getInstallType', () => { - it('should return correct type when installing and no other version is currently installed', () => {}); - const installTypeInstall = getInstallType({ pkgVersion: '1.0.0', installedPkg: undefined }); - expect(installTypeInstall).toBe('install'); + it('should return correct type when installing and no other version is currently installed', () => { + const installTypeInstall = getInstallType({ pkgVersion: '1.0.0', installedPkg: undefined }); + expect(installTypeInstall).toBe('install'); - it('should return correct type when installing the same version', () => {}); - const installTypeReinstall = getInstallType({ - pkgVersion: '1.0.0', - installedPkg: mockInstallation, + // @ts-expect-error can only be 'install' if no installedPkg given + expect(installTypeInstall === 'update').toBe(false); + // @ts-expect-error can only be 'install' if no installedPkg given + expect(installTypeInstall === 'reinstall').toBe(false); + // @ts-expect-error can only be 'install' if no installedPkg given + expect(installTypeInstall === 'reupdate').toBe(false); + // @ts-expect-error can only be 'install' if no installedPkg given + expect(installTypeInstall === 'rollback').toBe(false); }); - expect(installTypeReinstall).toBe('reinstall'); - it('should return correct type when moving from one version to another', () => {}); - const installTypeUpdate = getInstallType({ - pkgVersion: '1.0.1', - installedPkg: mockInstallation, + it('should return correct type when installing the same version', () => { + const installTypeReinstall = getInstallType({ + pkgVersion: '1.0.0', + installedPkg: mockInstallation, + }); + expect(installTypeReinstall).toBe('reinstall'); + + // @ts-expect-error cannot be 'install' if given installedPkg + expect(installTypeReinstall === 'install').toBe(false); + }); + + it('should return correct type when moving from one version to another', () => { + const installTypeUpdate = getInstallType({ + pkgVersion: '1.0.1', + installedPkg: mockInstallation, + }); + expect(installTypeUpdate).toBe('update'); + + // @ts-expect-error cannot be 'install' if given installedPkg + expect(installTypeUpdate === 'install').toBe(false); }); - expect(installTypeUpdate).toBe('update'); - it('should return correct type when update fails and trys again', () => {}); - const installTypeReupdate = getInstallType({ - pkgVersion: '1.0.1', - installedPkg: mockInstallationUpdateFail, + it('should return correct type when update fails and trys again', () => { + const installTypeReupdate = getInstallType({ + pkgVersion: '1.0.1', + installedPkg: mockInstallationUpdateFail, + }); + expect(installTypeReupdate).toBe('reupdate'); + + // @ts-expect-error cannot be 'install' if given installedPkg + expect(installTypeReupdate === 'install').toBe(false); }); - expect(installTypeReupdate).toBe('reupdate'); - it('should return correct type when attempting to rollback from a failed update', () => {}); - const installTypeRollback = getInstallType({ - pkgVersion: '1.0.0', - installedPkg: mockInstallationUpdateFail, + it('should return correct type when attempting to rollback from a failed update', () => { + const installTypeRollback = getInstallType({ + pkgVersion: '1.0.0', + installedPkg: mockInstallationUpdateFail, + }); + expect(installTypeRollback).toBe('rollback'); + + // @ts-expect-error cannot be 'install' if given installedPkg + expect(installTypeRollback === 'install').toBe(false); }); - expect(installTypeRollback).toBe('rollback'); }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index e6144e0309594..54b9c4d3fbb17 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -200,22 +200,20 @@ export async function installPackage({ ); // if this is an update or retrying an update, delete the previous version's pipelines - if (installType === 'update' || installType === 'reupdate') { + if ((installType === 'update' || installType === 'reupdate') && installedPkg) { await deletePreviousPipelines( callCluster, savedObjectsClient, pkgName, - // @ts-ignore installType conditions already check for existence of installedPkg installedPkg.attributes.version ); } // pipelines from a different version may have installed during a failed update - if (installType === 'rollback') { + if (installType === 'rollback' && installedPkg) { await deletePreviousPipelines( callCluster, savedObjectsClient, pkgName, - // @ts-ignore installType conditions already check for existence of installedPkg installedPkg.attributes.install_version ); } @@ -354,17 +352,32 @@ export async function ensurePackagesCompletedInstall( return installingPackages; } -export function getInstallType({ - pkgVersion, - installedPkg, -}: { +interface NoPkgArgs { pkgVersion: string; - installedPkg: SavedObject | undefined; -}): InstallType { - const isInstalledPkg = !!installedPkg; - const currentPkgVersion = installedPkg?.attributes.version; - const lastStartedInstallVersion = installedPkg?.attributes.install_version; - if (!isInstalledPkg) return 'install'; + installedPkg?: undefined; +} + +interface HasPkgArgs { + pkgVersion: string; + installedPkg: SavedObject; +} + +type OnlyInstall = Extract; +type NotInstall = Exclude; + +// overloads +export function getInstallType(args: NoPkgArgs): OnlyInstall; +export function getInstallType(args: HasPkgArgs): NotInstall; +export function getInstallType(args: NoPkgArgs | HasPkgArgs): OnlyInstall | NotInstall; + +// implementation +export function getInstallType(args: NoPkgArgs | HasPkgArgs): OnlyInstall | NotInstall { + const { pkgVersion, installedPkg } = args; + if (!installedPkg) return 'install'; + + const currentPkgVersion = installedPkg.attributes.version; + const lastStartedInstallVersion = installedPkg.attributes.install_version; + if (pkgVersion === currentPkgVersion && pkgVersion !== lastStartedInstallVersion) return 'rollback'; if (pkgVersion === currentPkgVersion) return 'reinstall'; From cd489e5f26df9560b7b763002f2e2187895ff0c7 Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Thu, 10 Sep 2020 13:27:25 -0400 Subject: [PATCH 10/29] [Security Solution][Endpoint][Admin]Task/kql bar only (#75066) --- .../endpoint_hosts/models/index_pattern.ts | 13 + .../pages/endpoint_hosts/store/action.ts | 12 + .../store/endpoint_pagination.test.ts | 1 + .../pages/endpoint_hosts/store/index.test.ts | 2 + .../endpoint_hosts/store/middleware.test.ts | 2 + .../pages/endpoint_hosts/store/middleware.ts | 41 +++- .../pages/endpoint_hosts/store/reducer.ts | 14 ++ .../pages/endpoint_hosts/store/selectors.ts | 39 ++- .../management/pages/endpoint_hosts/types.ts | 7 + .../view/components/search_bar.tsx | 70 ++++++ .../pages/endpoint_hosts/view/index.tsx | 35 ++- .../metadata/destination_index/data.json | 223 ++++++++++++++++++ .../apps/endpoint/endpoint_list.ts | 96 +++++++- 13 files changed, 536 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/models/index_pattern.ts create mode 100644 x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/search_bar.tsx create mode 100644 x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/models/index_pattern.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/models/index_pattern.ts new file mode 100644 index 0000000000000..064a591d0f3fa --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/models/index_pattern.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { all } from 'deepmerge'; +import { IIndexPattern } from '../../../../../../../../src/plugins/data/common'; +import { Immutable } from '../../../../../common/endpoint/types'; + +export function clone(value: IIndexPattern | Immutable): IIndexPattern { + return all([value]) as IIndexPattern; +} diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts index 5f36af2a2d8ea..84d09adfc295e 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/action.ts @@ -13,6 +13,7 @@ import { ServerApiError } from '../../../../common/types'; import { GetPolicyListResponse } from '../../policy/types'; import { GetPackagesResponse } from '../../../../../../ingest_manager/common'; import { EndpointState } from '../types'; +import { IIndexPattern } from '../../../../../../../../src/plugins/data/public'; interface ServerReturnedEndpointList { type: 'serverReturnedEndpointList'; @@ -86,6 +87,15 @@ interface ServerReturnedEndpointExistValue { payload: boolean; } +interface ServerReturnedMetadataPatterns { + type: 'serverReturnedMetadataPatterns'; + payload: IIndexPattern[]; +} + +interface ServerFailedToReturnMetadataPatterns { + type: 'serverFailedToReturnMetadataPatterns'; + payload: ServerApiError; +} interface UserUpdatedEndpointListRefreshOptions { type: 'userUpdatedEndpointListRefreshOptions'; payload: { @@ -112,6 +122,8 @@ export type EndpointAction = | ServerReturnedEndpointExistValue | ServerCancelledPolicyItemsLoading | ServerReturnedEndpointPackageInfo + | ServerReturnedMetadataPatterns + | ServerFailedToReturnMetadataPatterns | AppRequestedEndpointList | ServerReturnedEndpointNonExistingPolicies | UserUpdatedEndpointListRefreshOptions; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/endpoint_pagination.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/endpoint_pagination.test.ts index 0fd970f4bed12..b4e00319485e9 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/endpoint_pagination.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/endpoint_pagination.test.ts @@ -77,6 +77,7 @@ describe('endpoint list pagination: ', () => { expect(fakeHttpServices.post).toHaveBeenCalledWith('/api/endpoint/metadata', { body: JSON.stringify({ paging_properties: [{ page_index: '0' }, { page_size: '10' }], + filters: { kql: '' }, }), }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts index 3a095644b3b41..f28ae9bf55ab2 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/index.test.ts @@ -53,6 +53,8 @@ describe('EndpointList store concerns', () => { endpointPackageInfo: undefined, nonExistingPolicies: {}, endpointsExist: true, + patterns: [], + patternsError: undefined, isAutoRefreshEnabled: true, autoRefreshInterval: DEFAULT_POLL_INTERVAL, }); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts index 15e89f9771382..c4d2886f3e8e5 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts @@ -72,6 +72,7 @@ describe('endpoint list middleware', () => { expect(fakeHttpServices.post).toHaveBeenCalledWith('/api/endpoint/metadata', { body: JSON.stringify({ paging_properties: [{ page_index: '0' }, { page_size: '10' }], + filters: { kql: '' }, }), }); expect(listData(getState())).toEqual(apiResponse.hosts); @@ -100,6 +101,7 @@ describe('endpoint list middleware', () => { expect(fakeHttpServices.post).toHaveBeenCalledWith('/api/endpoint/metadata', { body: JSON.stringify({ paging_properties: [{ page_index: '0' }, { page_size: '10' }], + filters: { kql: '' }, }), }); expect(listData(getState())).toEqual(apiResponse.hosts); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts index 2650aa4865228..5bf085023c65d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts @@ -15,6 +15,8 @@ import { listData, endpointPackageInfo, nonExistingPolicies, + patterns, + searchBarQuery, } from './selectors'; import { EndpointState } from '../types'; import { @@ -23,8 +25,24 @@ import { sendGetAgentPolicyList, } from '../../policy/store/policy_list/services/ingest'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../../../../../ingest_manager/common'; +import { metadataCurrentIndexPattern } from '../../../../../common/endpoint/constants'; +import { IIndexPattern, Query } from '../../../../../../../../src/plugins/data/public'; -export const endpointMiddlewareFactory: ImmutableMiddlewareFactory = (coreStart) => { +export const endpointMiddlewareFactory: ImmutableMiddlewareFactory = ( + coreStart, + depsStart +) => { + async function fetchIndexPatterns(): Promise { + const { indexPatterns } = depsStart.data; + const fields = await indexPatterns.getFieldsForWildcard({ + pattern: metadataCurrentIndexPattern, + }); + const indexPattern: IIndexPattern = { + title: metadataCurrentIndexPattern, + fields, + }; + return [indexPattern]; + } // eslint-disable-next-line complexity return ({ getState, dispatch }) => (next) => async (action) => { next(action); @@ -52,10 +70,31 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory('/api/endpoint/metadata', { body: JSON.stringify({ paging_properties: [{ page_index: pageIndex }, { page_size: pageSize }], + filters: { kql: decodedQuery.query }, }), }); endpointResponse.request_page_index = Number(pageIndex); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts index 060321fa40401..d688fa3b76b5a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/reducer.ts @@ -31,6 +31,8 @@ export const initialEndpointListState: Immutable = { endpointPackageInfo: undefined, nonExistingPolicies: {}, endpointsExist: true, + patterns: [], + patternsError: undefined, isAutoRefreshEnabled: true, autoRefreshInterval: DEFAULT_POLL_INTERVAL, }; @@ -70,6 +72,18 @@ export const endpointListReducer: ImmutableReducer = ( ...action.payload, }, }; + } else if (action.type === 'serverReturnedMetadataPatterns') { + // handle error case + return { + ...state, + patterns: action.payload, + patternsError: undefined, + }; + } else if (action.type === 'serverFailedToReturnMetadataPatterns') { + return { + ...state, + patternsError: action.payload, + }; } else if (action.type === 'serverReturnedEndpointDetails') { return { ...state, diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts index e8abe37cf0a88..8eefcc271794a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/selectors.ts @@ -8,6 +8,7 @@ import querystring from 'querystring'; import { createSelector } from 'reselect'; import { matchPath } from 'react-router-dom'; +import { decode } from 'rison-node'; import { Immutable, HostPolicyResponseAppliedAction, @@ -21,6 +22,7 @@ import { MANAGEMENT_DEFAULT_PAGE_SIZE, MANAGEMENT_ROUTING_ENDPOINTS_PATH, } from '../../../common/constants'; +import { Query } from '../../../../../../../../src/plugins/data/common/query/types'; export const listData = (state: Immutable) => state.hosts; @@ -57,6 +59,13 @@ export const endpointPackageVersion = createSelector( (info) => info?.version ?? undefined ); +/** + * Returns the index patterns for the SearchBar to use for autosuggest + */ +export const patterns = (state: Immutable) => state.patterns; + +export const patternsError = (state: Immutable) => state.patternsError; + /** * Returns the full policy response from the endpoint after a user modifies a policy. */ @@ -142,7 +151,11 @@ export const uiQueryParams: ( const query = querystring.parse(location.search.slice(1)); const paginationParams = extractListPaginationParams(query); - const keys: Array = ['selected_endpoint', 'show']; + const keys: Array = [ + 'selected_endpoint', + 'show', + 'admin_query', + ]; for (const key of keys) { const value: string | undefined = @@ -210,3 +223,27 @@ export const nonExistingPolicies: ( */ export const endpointsExist: (state: Immutable) => boolean = (state) => state.endpointsExist; + +/** + * Returns query text from query bar + */ +export const searchBarQuery: (state: Immutable) => Query = createSelector( + uiQueryParams, + ({ admin_query: adminQuery }) => { + const decodedQuery: Query = { query: '', language: 'kuery' }; + if (adminQuery) { + const urlDecodedQuery = (decode(adminQuery) as unknown) as Query; + if (urlDecodedQuery && typeof urlDecodedQuery.query === 'string') { + decodedQuery.query = urlDecodedQuery.query; + } + if ( + urlDecodedQuery && + typeof urlDecodedQuery.language === 'string' && + (urlDecodedQuery.language === 'kuery' || urlDecodedQuery.language === 'lucene') + ) { + decodedQuery.language = urlDecodedQuery.language; + } + } + return decodedQuery; + } +); diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts index 5a6a1af7bd7e8..b73e60718d12e 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/types.ts @@ -14,6 +14,7 @@ import { } from '../../../../common/endpoint/types'; import { ServerApiError } from '../../../common/types'; import { GetPackagesResponse } from '../../../../../ingest_manager/common'; +import { IIndexPattern } from '../../../../../../../src/plugins/data/public'; export interface EndpointState { /** list of host **/ @@ -54,6 +55,10 @@ export interface EndpointState { nonExistingPolicies: Record; /** Tracks whether hosts exist and helps control if onboarding should be visible */ endpointsExist: boolean; + /** index patterns for query bar */ + patterns: IIndexPattern[]; + /** api error from retrieving index patters for query bar */ + patternsError?: ServerApiError; /** Is auto-refresh enabled */ isAutoRefreshEnabled: boolean; /** The current auto refresh interval for data in ms */ @@ -72,4 +77,6 @@ export interface EndpointIndexUIQueryParams { page_index?: string; /** show the policy response or host details */ show?: 'policy_response' | 'details'; + /** Query text from search bar*/ + admin_query?: string; } diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/search_bar.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/search_bar.tsx new file mode 100644 index 0000000000000..b6349a45f383d --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/components/search_bar.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { memo, useCallback, useMemo } from 'react'; +import { useHistory } from 'react-router-dom'; +import { encode, RisonValue } from 'rison-node'; +import styled from 'styled-components'; +import { Query, SearchBar, TimeHistory } from '../../../../../../../../../src/plugins/data/public'; +import { Storage } from '../../../../../../../../../src/plugins/kibana_utils/public'; +import { urlFromQueryParams } from '../url_from_query_params'; +import { useEndpointSelector } from '../hooks'; +import * as selectors from '../../store/selectors'; +import { clone } from '../../models/index_pattern'; + +const AdminQueryBar = styled.div` + .globalQueryBar { + padding: 0; + } +`; + +export const AdminSearchBar = memo(() => { + const history = useHistory(); + const queryParams = useEndpointSelector(selectors.uiQueryParams); + const searchBarIndexPatterns = useEndpointSelector(selectors.patterns); + const searchBarQuery = useEndpointSelector(selectors.searchBarQuery); + const clonedIndexPatterns = useMemo( + () => searchBarIndexPatterns.map((pattern) => clone(pattern)), + [searchBarIndexPatterns] + ); + + const onQuerySubmit = useCallback( + (params: { query?: Query }) => { + history.push( + urlFromQueryParams({ + ...queryParams, + admin_query: encode((params.query as unknown) as RisonValue), + }) + ); + }, + [history, queryParams] + ); + + const timeHistory = useMemo(() => new TimeHistory(new Storage(localStorage)), []); + + return ( +
+ {searchBarIndexPatterns && searchBarIndexPatterns.length > 0 && ( + + + + )} +
+ ); +}); + +AdminSearchBar.displayName = 'AdminSearchBar'; diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx index a569c4f02604b..378f3cc4cb316 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx @@ -16,6 +16,8 @@ import { EuiSelectableProps, EuiSuperDatePicker, EuiSpacer, + EuiFlexGroup, + EuiFlexItem, } from '@elastic/eui'; import { useHistory } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; @@ -46,6 +48,7 @@ import { getEndpointListPath, getEndpointDetailsPath } from '../../../common/rou import { useFormatUrl } from '../../../../common/components/link_to'; import { EndpointAction } from '../store/action'; import { EndpointPolicyLink } from './components/endpoint_policy_link'; +import { AdminSearchBar } from './components/search_bar'; import { AdministrationListPage } from '../../../components/administration_list_page'; const EndpointListNavLink = memo<{ @@ -89,6 +92,7 @@ export const EndpointList = () => { endpointsExist, autoRefreshInterval, isAutoRefreshEnabled, + patternsError, } = useEndpointSelector(selector); const { formatUrl, search } = useFormatUrl(SecurityPageName.administration); @@ -397,16 +401,16 @@ export const EndpointList = () => { const hasListData = listData && listData.length > 0; const refreshStyle = useMemo(() => { - return { display: hasListData ? 'flex' : 'none', maxWidth: 200 }; - }, [hasListData]); + return { display: endpointsExist ? 'flex' : 'none', maxWidth: 200 }; + }, [endpointsExist]); const refreshIsPaused = useMemo(() => { - return !hasListData ? false : hasSelectedEndpoint ? true : !isAutoRefreshEnabled; - }, [hasListData, hasSelectedEndpoint, isAutoRefreshEnabled]); + return !endpointsExist ? false : hasSelectedEndpoint ? true : !isAutoRefreshEnabled; + }, [endpointsExist, hasSelectedEndpoint, isAutoRefreshEnabled]); const refreshInterval = useMemo(() => { - return !hasListData ? DEFAULT_POLL_INTERVAL : autoRefreshInterval; - }, [hasListData, autoRefreshInterval]); + return !endpointsExist ? DEFAULT_POLL_INTERVAL : autoRefreshInterval; + }, [endpointsExist, autoRefreshInterval]); return ( { } > {hasSelectedEndpoint && } - { - <> -
+ <> + + {endpointsExist && !patternsError && ( + + + + )} + { onRefreshChange={onRefreshChange} isAutoRefreshOnly={true} /> -
- - - } + + + + {hasListData && ( <> diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json new file mode 100644 index 0000000000000..b19e5e2cbf1d7 --- /dev/null +++ b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json @@ -0,0 +1,223 @@ +{ + "type": "doc", + "value": { + "id": "M92ScEJT9M9QusfIi3hpEb0AAAAAAAAA", + "index": "metrics-endpoint.metadata_current-default", + "source": { + "HostDetails": { + "@timestamp": 1579881969541, + "Endpoint": { + "policy": { + "applied": { + "id": "00000000-0000-0000-0000-000000000000", + "name": "Default", + "status": "failure" + } + }, + "status": "enrolled" + }, + "agent": { + "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", + "name": "Elastic Endpoint", + "version": "6.8.0" + }, + "elastic": { + "agent": { + "id": "023fa40c-411d-4188-a941-4147bfadd095" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1579881969541, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", + "ingested": "2020-09-09T18:25:15.853783Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "hostname": "rezzani-7.example.com", + "id": "fc0ff548-feba-41b6-8367-65e8790d0eaf", + "ip": [ + "10.101.149.26", + "2606:a000:ffc0:39:11ef:37b9:3371:578c" + ], + "mac": [ + "e2-6d-f9-0-46-2e" + ], + "name": "rezzani-7.example.com", + "os": { + "Ext": { + "variant": "Windows Pro" + }, + "family": "Windows", + "full": "Windows 10", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } + } + }, + "agent": { + "id": "3838df35-a095-4af4-8fce-0b6d78793f2e" + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "OU3RgCJaNnR90byeDEHutp8AAAAAAAAA", + "index": "metrics-endpoint.metadata_current-default", + "source": { + "HostDetails": { + "@timestamp": 1579881969541, + "Endpoint": { + "policy": { + "applied": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", + "name": "Default", + "status": "failure" + } + }, + "status": "enrolled" + }, + "agent": { + "id": "963b081e-60d1-482c-befd-a5815fa8290f", + "name": "Elastic Endpoint", + "version": "6.6.1" + }, + "elastic": { + "agent": { + "id": "11488bae-880b-4e7b-8d28-aac2aa9de816" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1579881969541, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", + "ingested": "2020-09-09T18:25:14.919526Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86", + "hostname": "cadmann-4.example.com", + "id": "1fb3e58f-6ab0-4406-9d2a-91911207a712", + "ip": [ + "10.192.213.130", + "10.70.28.129" + ], + "mac": [ + "a9-71-6a-cc-93-85", + "f7-31-84-d3-21-68", + "2-95-12-39-ca-71" + ], + "name": "cadmann-4.example.com", + "os": { + "Ext": { + "variant": "Windows Pro" + }, + "family": "Windows", + "full": "Windows 10", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } + } + }, + "agent": { + "id": "963b081e-60d1-482c-befd-a5815fa8290f" + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "YjqDCEuI6JmLeLOSyZx_NhMAAAAAAAAA", + "index": "metrics-endpoint.metadata_current-default", + "source": { + "HostDetails": { + "@timestamp": 1579881969541, + "Endpoint": { + "policy": { + "applied": { + "id": "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", + "name": "Default", + "status": "success" + } + }, + "status": "enrolled" + }, + "agent": { + "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", + "name": "Elastic Endpoint", + "version": "6.0.0" + }, + "elastic": { + "agent": { + "id": "92ac1ce0-e1f7-409e-8af6-f17e97b1fc71" + } + }, + "event": { + "action": "endpoint_metadata", + "category": [ + "host" + ], + "created": 1579881969541, + "dataset": "endpoint.metadata", + "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", + "ingested": "2020-09-09T18:25:15.853404Z", + "kind": "metric", + "module": "endpoint", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "hostname": "thurlow-9.example.com", + "id": "2f735e3d-be14-483b-9822-bad06e9045ca", + "ip": [ + "10.46.229.234" + ], + "mac": [ + "30-8c-45-55-69-b8", + "e5-36-7e-8f-a3-84", + "39-a1-37-20-18-74" + ], + "name": "thurlow-9.example.com", + "os": { + "Ext": { + "variant": "Windows Server" + }, + "family": "Windows", + "full": "Windows Server 2016", + "name": "windows 10.0", + "platform": "Windows", + "version": "10.0" + } + } + }, + "agent": { + "id": "b3412d6f-b022-4448-8fee-21cc936ea86b" + } + } + } +} diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index ebd5ff0afee77..00b4b82f9d602 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; import { deleteMetadataCurrentStream, deleteMetadataStream, + deleteAllDocsFromMetadataCurrentIndex, } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper'; export default ({ getPageObjects, getService }: FtrProviderContext) => { @@ -68,11 +69,13 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { before(async () => { await deleteMetadataStream(getService); await deleteMetadataCurrentStream(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); await pageObjects.endpoint.navigateToEndpointList(); }); after(async () => { await deleteMetadataStream(getService); await deleteMetadataCurrentStream(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); }); it('finds no data in list and prompts onboarding to add policy', async () => { @@ -80,8 +83,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('finds data after load and polling', async () => { - await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true }); - await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 120000); + await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true }); + await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 1100); const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable'); expect(tableData).to.eql(expectedData); }); @@ -89,13 +92,13 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('when there is data,', () => { before(async () => { - await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true }); - await sleep(120000); + await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true }); await pageObjects.endpoint.navigateToEndpointList(); }); after(async () => { await deleteMetadataStream(getService); await deleteMetadataCurrentStream(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); }); it('finds page title', async () => { @@ -212,6 +215,91 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); }); + describe('displays the correct table data for the kql queries', () => { + before(async () => { + await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true }); + await pageObjects.endpoint.navigateToEndpointList(); + }); + after(async () => { + await deleteMetadataStream(getService); + await deleteMetadataCurrentStream(getService); + await deleteAllDocsFromMetadataCurrentIndex(getService); + }); + it('for the kql query: na, table shows an empty list', async () => { + await testSubjects.setValue('adminSearchBar', 'na'); + await (await testSubjects.find('querySubmitButton')).click(); + const expectedDataFromQuery = [ + [ + 'Hostname', + 'Agent Status', + 'Integration', + 'Configuration Status', + 'Operating System', + 'IP Address', + 'Version', + 'Last Active', + ], + ['No items found'], + ]; + + await pageObjects.endpoint.waitForTableToNotHaveData('endpointListTable'); + const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable'); + expect(tableData).to.eql(expectedDataFromQuery); + }); + + it('for the kql query: HostDetails.Endpoint.policy.applied.id : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A", table shows 2 items', async () => { + await testSubjects.setValue('adminSearchBar', ' '); + await (await testSubjects.find('querySubmitButton')).click(); + + const endpointListTableTotal = await testSubjects.getVisibleText('endpointListTableTotal'); + + await testSubjects.setValue( + 'adminSearchBar', + 'HostDetails.Endpoint.policy.applied.id : "C2A9093E-E289-4C0A-AA44-8C32A414FA7A" ' + ); + await (await testSubjects.find('querySubmitButton')).click(); + const expectedDataFromQuery = [ + [ + 'Hostname', + 'Agent Status', + 'Integration', + 'Configuration Status', + 'Operating System', + 'IP Address', + 'Version', + 'Last Active', + ], + [ + 'cadmann-4.example.com', + 'Error', + 'Default', + 'Failure', + 'windows 10.0', + '10.192.213.130, 10.70.28.129', + '6.6.1', + 'Jan 24, 2020 @ 16:06:09.541', + ], + [ + 'thurlow-9.example.com', + 'Error', + 'Default', + 'Success', + 'windows 10.0', + '10.46.229.234', + '6.0.0', + 'Jan 24, 2020 @ 16:06:09.541', + ], + ]; + + await pageObjects.endpoint.waitForVisibleTextToChange( + 'endpointListTableTotal', + endpointListTableTotal + ); + const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable'); + expect(tableData).to.eql(expectedDataFromQuery); + }); + }); + describe.skip('when there is no data,', () => { before(async () => { // clear out the data and reload the page From 5d12eda2d56db97d8ffed9ffcf517a5950d63a3f Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Thu, 10 Sep 2020 13:33:27 -0400 Subject: [PATCH 11/29] [Input Controls] Fix Resize Resetting Selections (#76573) Fixed resizing controls visualization resetting selections --- .../public/vis_controller.tsx | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/plugins/input_control_vis/public/vis_controller.tsx b/src/plugins/input_control_vis/public/vis_controller.tsx index e4310960851ca..faea98b792291 100644 --- a/src/plugins/input_control_vis/public/vis_controller.tsx +++ b/src/plugins/input_control_vis/public/vis_controller.tsx @@ -18,8 +18,10 @@ */ import React from 'react'; +import { isEqual } from 'lodash'; import { render, unmountComponentAtNode } from 'react-dom'; +import { Subscription } from 'rxjs'; import { I18nStart } from 'kibana/public'; import { InputControlVis } from './components/vis/input_control_vis'; import { getControlFactory } from './control/control_factory'; @@ -34,11 +36,13 @@ import { VisParams, Vis } from '../../visualizations/public'; export const createInputControlVisController = (deps: InputControlVisDependencies) => { return class InputControlVisController { private I18nContext?: I18nStart['Context']; + private isLoaded = false; controls: Array; queryBarUpdateHandler: () => void; filterManager: FilterManager; updateSubsciption: any; + timeFilterSubscription: Subscription; visParams?: VisParams; constructor(public el: Element, public vis: Vis) { @@ -50,19 +54,32 @@ export const createInputControlVisController = (deps: InputControlVisDependencie this.updateSubsciption = this.filterManager .getUpdates$() .subscribe(this.queryBarUpdateHandler); + this.timeFilterSubscription = deps.data.query.timefilter.timefilter + .getTimeUpdate$() + .subscribe(() => { + if (this.visParams?.useTimeFilter) { + this.isLoaded = false; + } + }); } async render(visData: any, visParams: VisParams) { - this.visParams = visParams; - this.controls = []; - this.controls = await this.initControls(); - const [{ i18n }] = await deps.core.getStartServices(); - this.I18nContext = i18n.Context; + if (!this.I18nContext) { + const [{ i18n }] = await deps.core.getStartServices(); + this.I18nContext = i18n.Context; + } + if (!this.isLoaded || !isEqual(visParams, this.visParams)) { + this.visParams = visParams; + this.controls = []; + this.controls = await this.initControls(); + this.isLoaded = true; + } this.drawVis(); } destroy() { this.updateSubsciption.unsubscribe(); + this.timeFilterSubscription.unsubscribe(); unmountComponentAtNode(this.el); this.controls.forEach((control) => control.destroy()); } From eacd602612fc80f8aa691988fcf7bcdd34607cef Mon Sep 17 00:00:00 2001 From: Chris Roberson Date: Thu, 10 Sep 2020 13:50:16 -0400 Subject: [PATCH 12/29] Use proper lodash syntax (#77105) Co-authored-by: Elastic Machine --- .../public/components/chart/monitoring_timeseries.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring/public/components/chart/monitoring_timeseries.js b/x-pack/plugins/monitoring/public/components/chart/monitoring_timeseries.js index deaa4fd152cce..c4faf51dc000c 100644 --- a/x-pack/plugins/monitoring/public/components/chart/monitoring_timeseries.js +++ b/x-pack/plugins/monitoring/public/components/chart/monitoring_timeseries.js @@ -11,8 +11,8 @@ import { getColor } from './get_color'; import { TimeseriesVisualization } from './timeseries_visualization'; function formatTicksFor(series) { - const format = get(series, '.metric.format', '0,0.0'); - const units = get(series, '.metric.units', ''); + const format = get(series, 'metric.format', '0,0.0'); + const units = get(series, 'metric.units', ''); return function formatTicks(val) { let formatted = numeral(val).format(format); From e7b02d06cc023528ab868891fca8e523bb390917 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Thu, 10 Sep 2020 14:26:35 -0400 Subject: [PATCH 13/29] [Security Solution] Use safe type in resolver backend (#76969) * Moving generator to safe type version * Finished generator and alert * Gzipping again * Finishing type conversions for backend * Trying to cast front end tests back to unsafe type for now * Working reducer tests * Adding more comments and fixing alert type * Restoring resolver test data * Updating snapshot with timestamp info * Removing todo and fixing test Co-authored-by: Elastic Machine --- .../common/endpoint/generate_data.test.ts | 142 ++++---- .../common/endpoint/generate_data.ts | 119 ++++--- .../common/endpoint/index_data.ts | 3 +- .../endpoint/models/ecs_safety_helpers.ts | 4 +- .../common/endpoint/models/event.test.ts | 41 ++- .../common/endpoint/models/event.ts | 55 +-- .../common/endpoint/types/index.ts | 322 +++++++++++------- .../isometric_taxi_layout.test.ts.snap | 36 ++ .../resolver/store/data/reducer.test.ts | 34 +- .../routes/resolver/queries/alerts.ts | 6 +- .../endpoint/routes/resolver/queries/base.ts | 8 +- .../routes/resolver/queries/children.ts | 6 +- .../routes/resolver/queries/events.ts | 6 +- .../routes/resolver/queries/lifecycle.ts | 6 +- .../routes/resolver/queries/multi_searcher.ts | 6 +- .../endpoint/routes/resolver/queries/stats.ts | 4 +- .../resolver/utils/alerts_query_handler.ts | 4 +- .../resolver/utils/ancestry_query_handler.ts | 49 +-- .../resolver/utils/children_helper.test.ts | 10 +- .../routes/resolver/utils/children_helper.ts | 37 +- .../utils/children_lifecycle_query_handler.ts | 10 +- .../resolver/utils/children_pagination.ts | 8 +- .../utils/children_start_query_handler.ts | 4 +- .../resolver/utils/events_query_handler.ts | 8 +- .../endpoint/routes/resolver/utils/fetch.ts | 16 +- .../resolver/utils/lifecycle_query_handler.ts | 10 +- .../endpoint/routes/resolver/utils/node.ts | 34 +- .../routes/resolver/utils/pagination.test.ts | 10 +- .../routes/resolver/utils/pagination.ts | 19 +- .../routes/resolver/utils/tree.test.ts | 24 +- .../endpoint/routes/resolver/utils/tree.ts | 28 +- .../apis/resolver/alerts.ts | 19 +- .../apis/resolver/children.ts | 98 +++--- .../apis/resolver/common.ts | 55 ++- .../apis/resolver/entity_id.ts | 46 +-- .../apis/resolver/events.ts | 28 +- .../apis/resolver/tree.ts | 58 ++-- .../services/resolver.ts | 3 +- 38 files changed, 794 insertions(+), 582 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts index be3a1e82356c8..7e3b3d125fb5d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts @@ -13,6 +13,12 @@ import { ECSCategory, ANCESTRY_LIMIT, } from './generate_data'; +import { firstNonNullValue, values } from './models/ecs_safety_helpers'; +import { + entityIDSafeVersion, + parentEntityIDSafeVersion, + timestampSafeVersion, +} from './models/event'; interface Node { events: Event[]; @@ -30,7 +36,7 @@ describe('data generator', () => { const event1 = generator.generateEvent(); const event2 = generator.generateEvent(); - expect(event2.event.sequence).toBe(event1.event.sequence + 1); + expect(event2.event?.sequence).toBe((firstNonNullValue(event1.event?.sequence) ?? 0) + 1); }); it('creates the same documents with same random seed', () => { @@ -76,37 +82,37 @@ describe('data generator', () => { const timestamp = new Date().getTime(); const alert = generator.generateAlert(timestamp); expect(alert['@timestamp']).toEqual(timestamp); - expect(alert.event.action).not.toBeNull(); + expect(alert.event?.action).not.toBeNull(); expect(alert.Endpoint).not.toBeNull(); expect(alert.agent).not.toBeNull(); expect(alert.host).not.toBeNull(); - expect(alert.process.entity_id).not.toBeNull(); + expect(alert.process?.entity_id).not.toBeNull(); }); it('creates process event documents', () => { const timestamp = new Date().getTime(); const processEvent = generator.generateEvent({ timestamp }); expect(processEvent['@timestamp']).toEqual(timestamp); - expect(processEvent.event.category).toEqual(['process']); - expect(processEvent.event.kind).toEqual('event'); - expect(processEvent.event.type).toEqual(['start']); + expect(processEvent.event?.category).toEqual(['process']); + expect(processEvent.event?.kind).toEqual('event'); + expect(processEvent.event?.type).toEqual(['start']); expect(processEvent.agent).not.toBeNull(); expect(processEvent.host).not.toBeNull(); - expect(processEvent.process.entity_id).not.toBeNull(); - expect(processEvent.process.name).not.toBeNull(); + expect(processEvent.process?.entity_id).not.toBeNull(); + expect(processEvent.process?.name).not.toBeNull(); }); it('creates other event documents', () => { const timestamp = new Date().getTime(); const processEvent = generator.generateEvent({ timestamp, eventCategory: 'dns' }); expect(processEvent['@timestamp']).toEqual(timestamp); - expect(processEvent.event.category).toEqual('dns'); - expect(processEvent.event.kind).toEqual('event'); - expect(processEvent.event.type).toEqual(['start']); + expect(processEvent.event?.category).toEqual('dns'); + expect(processEvent.event?.kind).toEqual('event'); + expect(processEvent.event?.type).toEqual(['start']); expect(processEvent.agent).not.toBeNull(); expect(processEvent.host).not.toBeNull(); - expect(processEvent.process.entity_id).not.toBeNull(); - expect(processEvent.process.name).not.toBeNull(); + expect(processEvent.process?.entity_id).not.toBeNull(); + expect(processEvent.process?.name).not.toBeNull(); }); describe('creates events with an empty ancestry array', () => { @@ -128,7 +134,7 @@ describe('data generator', () => { it('creates all events with an empty ancestry array', () => { for (const event of tree.allEvents) { - expect(event.process.Ext!.ancestry!.length).toEqual(0); + expect(event.process?.Ext?.ancestry?.length).toEqual(0); } }); }); @@ -194,22 +200,23 @@ describe('data generator', () => { const inRelated = node.relatedEvents.includes(event); const inRelatedAlerts = node.relatedAlerts.includes(event); - return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id; + return (inRelated || inRelatedAlerts || inLifecycle) && event.process?.entity_id === node.id; }; const verifyAncestry = (event: Event, genTree: Tree) => { - if (event.process.Ext!.ancestry!.length > 0) { - expect(event.process.parent?.entity_id).toBe(event.process.Ext!.ancestry![0]); + const ancestry = values(event.process?.Ext?.ancestry); + if (ancestry.length > 0) { + expect(event.process?.parent?.entity_id).toBe(ancestry[0]); } - for (let i = 0; i < event.process.Ext!.ancestry!.length; i++) { - const ancestor = event.process.Ext!.ancestry![i]; + for (let i = 0; i < ancestry.length; i++) { + const ancestor = ancestry[i]; const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor); - expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id); + expect(ancestor).toBe(parent?.lifecycle[0].process?.entity_id); // the next ancestor should be the grandparent - if (i + 1 < event.process.Ext!.ancestry!.length) { - const grandparent = event.process.Ext!.ancestry![i + 1]; - expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id); + if (i + 1 < ancestry.length) { + const grandparent = ancestry[i + 1]; + expect(grandparent).toBe(parent?.lifecycle[0].process?.parent?.entity_id); } } }; @@ -217,13 +224,14 @@ describe('data generator', () => { it('creates related events in ascending order', () => { // the order should not change since it should already be in ascending order const relatedEventsAsc = _.cloneDeep(tree.origin.relatedEvents).sort( - (event1, event2) => event1['@timestamp'] - event2['@timestamp'] + (event1, event2) => + (timestampSafeVersion(event1) ?? 0) - (timestampSafeVersion(event2) ?? 0) ); expect(tree.origin.relatedEvents).toStrictEqual(relatedEventsAsc); }); it('has ancestry array defined', () => { - expect(tree.origin.lifecycle[0].process.Ext!.ancestry!.length).toBe(ANCESTRY_LIMIT); + expect(values(tree.origin.lifecycle[0].process?.Ext?.ancestry).length).toBe(ANCESTRY_LIMIT); for (const event of tree.allEvents) { verifyAncestry(event, tree); } @@ -252,12 +260,9 @@ describe('data generator', () => { const counts: Record = {}; for (const event of node.relatedEvents) { - if (Array.isArray(event.event.category)) { - for (const cat of event.event.category) { - counts[cat] = counts[cat] + 1 || 1; - } - } else { - counts[event.event.category] = counts[event.event.category] + 1 || 1; + const categories = values(event.event?.category); + for (const cat of categories) { + counts[cat] = counts[cat] + 1 || 1; } } expect(counts[ECSCategory.Driver]).toEqual(1); @@ -316,15 +321,18 @@ describe('data generator', () => { expect(tree.allEvents.length).toBeGreaterThan(0); tree.allEvents.forEach((event) => { - const ancestor = tree.ancestry.get(event.process.entity_id); - if (ancestor) { - expect(eventInNode(event, ancestor)).toBeTruthy(); - return; - } + const entityID = entityIDSafeVersion(event); + if (entityID) { + const ancestor = tree.ancestry.get(entityID); + if (ancestor) { + expect(eventInNode(event, ancestor)).toBeTruthy(); + return; + } - const children = tree.children.get(event.process.entity_id); - if (children) { - expect(eventInNode(event, children)).toBeTruthy(); + const children = tree.children.get(entityID); + if (children) { + expect(eventInNode(event, children)).toBeTruthy(); + } } }); }); @@ -351,9 +359,8 @@ describe('data generator', () => { let events: Event[]; const isCategoryProcess = (event: Event) => { - return ( - _.isEqual(event.event.category, ['process']) || _.isEqual(event.event.category, 'process') - ); + const category = values(event.event?.category); + return _.isEqual(category, ['process']); }; beforeEach(() => { @@ -366,12 +373,16 @@ describe('data generator', () => { it('with n-1 process events', () => { for (let i = events.length - 2; i > 0; ) { - const parentEntityIdOfChild = events[i].process.parent?.entity_id; - for (; --i >= -1 && (events[i].event.kind !== 'event' || !isCategoryProcess(events[i])); ) { + const parentEntityIdOfChild = parentEntityIDSafeVersion(events[i]); + for ( + ; + --i >= -1 && (events[i].event?.kind !== 'event' || !isCategoryProcess(events[i])); + + ) { // related event - skip it } expect(i).toBeGreaterThanOrEqual(0); - expect(parentEntityIdOfChild).toEqual(events[i].process.entity_id); + expect(parentEntityIdOfChild).toEqual(entityIDSafeVersion(events[i])); } }); @@ -380,7 +391,7 @@ describe('data generator', () => { for ( ; previousProcessEventIndex >= -1 && - (events[previousProcessEventIndex].event.kind !== 'event' || + (events[previousProcessEventIndex].event?.kind !== 'event' || !isCategoryProcess(events[previousProcessEventIndex])); previousProcessEventIndex-- ) { @@ -388,14 +399,14 @@ describe('data generator', () => { } expect(previousProcessEventIndex).toBeGreaterThanOrEqual(0); // The alert should be last and have the same entity_id as the previous process event - expect(events[events.length - 1].process.entity_id).toEqual( - events[previousProcessEventIndex].process.entity_id + expect(events[events.length - 1].process?.entity_id).toEqual( + events[previousProcessEventIndex].process?.entity_id ); - expect(events[events.length - 1].process.parent?.entity_id).toEqual( - events[previousProcessEventIndex].process.parent?.entity_id + expect(events[events.length - 1].process?.parent?.entity_id).toEqual( + events[previousProcessEventIndex].process?.parent?.entity_id ); - expect(events[events.length - 1].event.kind).toEqual('alert'); - expect(events[events.length - 1].event.category).toEqual('malware'); + expect(events[events.length - 1].event?.kind).toEqual('alert'); + expect(events[events.length - 1].event?.category).toEqual('malware'); }); }); @@ -403,14 +414,17 @@ describe('data generator', () => { // First pass we gather up all the events by entity_id const tree: Record = {}; events.forEach((event) => { - if (event.process.entity_id in tree) { - tree[event.process.entity_id].events.push(event); - } else { - tree[event.process.entity_id] = { - events: [event], - children: [], - parent_entity_id: event.process.parent?.entity_id, - }; + const entityID = entityIDSafeVersion(event); + if (entityID) { + if (entityID in tree) { + tree[entityID].events.push(event); + } else { + tree[entityID] = { + events: [event], + children: [], + parent_entity_id: parentEntityIDSafeVersion(event), + }; + } } }); // Second pass add child references to each node @@ -419,8 +433,14 @@ describe('data generator', () => { tree[value.parent_entity_id].children.push(value); } } + + const entityID = entityIDSafeVersion(events[0]); + if (!entityID) { + throw new Error('entity id was invalid'); + } + // The root node must be first in the array or this fails - return tree[events[0].process.entity_id]; + return tree[entityID]; } function countResolverEvents(rootNode: Node, generations: number): number { diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index e1ff34463d215..7f31c71fe712b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -7,7 +7,6 @@ import uuid from 'uuid'; import seedrandom from 'seedrandom'; import { AlertEvent, - EndpointEvent, EndpointStatus, Host, HostMetadata, @@ -15,9 +14,15 @@ import { HostPolicyResponseActionStatus, OSFields, PolicyData, + SafeEndpointEvent, } from './types'; import { factory as policyFactory } from './models/policy_config'; -import { parentEntityId } from './models/event'; +import { + ancestryArray, + entityIDSafeVersion, + parentEntityIDSafeVersion, + timestampSafeVersion, +} from './models/event'; import { GetAgentPoliciesResponseItem, GetPackagesResponse, @@ -28,8 +33,9 @@ import { InstallationStatus, KibanaAssetReference, } from '../../../ingest_manager/common/types/models'; +import { firstNonNullValue } from './models/ecs_safety_helpers'; -export type Event = AlertEvent | EndpointEvent; +export type Event = AlertEvent | SafeEndpointEvent; /** * This value indicates the limit for the size of the ancestry array. The endpoint currently saves up to 20 values * in its messages. To simulate a limit on the array size I'm using 2 here so that we can't rely on there being a large @@ -426,13 +432,13 @@ export class EndpointDocGenerator { * @param ts - Timestamp to put in the event * @param entityID - entityID of the originating process * @param parentEntityID - optional entityID of the parent process, if it exists - * @param ancestryArray - an array of ancestors for the generated alert + * @param ancestry - an array of ancestors for the generated alert */ public generateAlert( ts = new Date().getTime(), entityID = this.randomString(10), parentEntityID?: string, - ancestryArray: string[] = [] + ancestry: string[] = [] ): AlertEvent { return { ...this.commonInfo, @@ -493,7 +499,7 @@ export class EndpointDocGenerator { sha256: 'fake sha256', }, Ext: { - ancestry: ancestryArray, + ancestry, code_signature: [ { trusted: false, @@ -555,7 +561,7 @@ export class EndpointDocGenerator { * Creates an event, customized by the options parameter * @param options - Allows event field values to be specified */ - public generateEvent(options: EventOptions = {}): EndpointEvent { + public generateEvent(options: EventOptions = {}): Event { // this will default to an empty array for the ancestry field if options.ancestry isn't included const ancestry: string[] = options.ancestry?.slice(0, options?.ancestryArrayLimit ?? ANCESTRY_LIMIT) ?? []; @@ -643,7 +649,11 @@ export class EndpointDocGenerator { public generateTree(options: TreeOptions = {}): Tree { const optionsWithDef = getTreeOptionsWithDef(options); const addEventToMap = (nodeMap: Map, event: Event) => { - const nodeId = event.process.entity_id; + const nodeId = entityIDSafeVersion(event); + if (!nodeId) { + return nodeMap; + } + // if a node already exists for the entity_id we'll use that one, otherwise let's create a new empty node // and add the event to the right array. let node = nodeMap.get(nodeId); @@ -652,18 +662,13 @@ export class EndpointDocGenerator { } // place the event in the right array depending on its category - if (event.event.kind === 'event') { - if ( - (Array.isArray(event.event.category) && - event.event.category.length === 1 && - event.event.category[0] === 'process') || - event.event.category === 'process' - ) { + if (firstNonNullValue(event.event?.kind) === 'event') { + if (firstNonNullValue(event.event?.category) === 'process') { node.lifecycle.push(event); } else { node.relatedEvents.push(event); } - } else if (event.event.kind === 'alert') { + } else if (firstNonNullValue(event.event?.kind) === 'alert') { node.relatedAlerts.push(event); } @@ -673,7 +678,7 @@ export class EndpointDocGenerator { const groupNodesByParent = (children: Map) => { const nodesByParent: Map> = new Map(); for (const node of children.values()) { - const parentID = parentEntityId(node.lifecycle[0]); + const parentID = parentEntityIDSafeVersion(node.lifecycle[0]); if (parentID) { let groupedNodes = nodesByParent.get(parentID); @@ -715,9 +720,13 @@ export class EndpointDocGenerator { const ancestryNodes: Map = ancestry.reduce(addEventToMap, new Map()); const alert = ancestry[ancestry.length - 1]; - const origin = ancestryNodes.get(alert.process.entity_id); + const alertEntityID = entityIDSafeVersion(alert); + if (!alertEntityID) { + throw Error("could not find the originating alert's entity id"); + } + const origin = ancestryNodes.get(alertEntityID); if (!origin) { - throw Error(`could not find origin while building tree: ${alert.process.entity_id}`); + throw Error(`could not find origin while building tree: ${alertEntityID}`); } const children = Array.from(this.descendantsTreeGenerator(alert, optionsWithDef)); @@ -799,7 +808,7 @@ export class EndpointDocGenerator { }); events.push(root); let ancestor = root; - let timestamp = root['@timestamp'] + 1000; + let timestamp = (timestampSafeVersion(root) ?? 0) + 1000; const addRelatedAlerts = ( node: Event, @@ -836,8 +845,8 @@ export class EndpointDocGenerator { events.push( this.generateEvent({ timestamp: timestamp + termProcessDuration * 1000, - entityID: root.process.entity_id, - parentEntityID: root.process.parent?.entity_id, + entityID: entityIDSafeVersion(root), + parentEntityID: parentEntityIDSafeVersion(root), eventCategory: ['process'], eventType: ['end'], }) @@ -845,13 +854,20 @@ export class EndpointDocGenerator { } for (let i = 0; i < opts.ancestors; i++) { + const ancestorEntityID = entityIDSafeVersion(ancestor); + const ancestry: string[] = []; + if (ancestorEntityID) { + ancestry.push(ancestorEntityID); + } + + ancestry.push(...(ancestryArray(ancestor) ?? [])); ancestor = this.generateEvent({ timestamp, - parentEntityID: ancestor.process.entity_id, + parentEntityID: entityIDSafeVersion(ancestor), // add the parent to the ancestry array - ancestry: [ancestor.process.entity_id, ...(ancestor.process.Ext?.ancestry ?? [])], + ancestry, ancestryArrayLimit: opts.ancestryArraySize, - parentPid: ancestor.process.pid, + parentPid: firstNonNullValue(ancestor.process?.pid), pid: this.randomN(5000), }); events.push(ancestor); @@ -862,11 +878,11 @@ export class EndpointDocGenerator { events.push( this.generateEvent({ timestamp: timestamp + termProcessDuration * 1000, - entityID: ancestor.process.entity_id, - parentEntityID: ancestor.process.parent?.entity_id, + entityID: entityIDSafeVersion(ancestor), + parentEntityID: parentEntityIDSafeVersion(ancestor), eventCategory: ['process'], eventType: ['end'], - ancestry: ancestor.process.Ext?.ancestry, + ancestry: ancestryArray(ancestor), ancestryArrayLimit: opts.ancestryArraySize, }) ); @@ -890,9 +906,9 @@ export class EndpointDocGenerator { events.push( this.generateAlert( timestamp, - ancestor.process.entity_id, - ancestor.process.parent?.entity_id, - ancestor.process.Ext?.ancestry + entityIDSafeVersion(ancestor), + parentEntityIDSafeVersion(ancestor), + ancestryArray(ancestor) ) ); return events; @@ -922,7 +938,7 @@ export class EndpointDocGenerator { maxChildren, }; const lineage: NodeState[] = [rootState]; - let timestamp = root['@timestamp']; + let timestamp = timestampSafeVersion(root) ?? 0; while (lineage.length > 0) { const currentState = lineage[lineage.length - 1]; // If we get to a state node and it has made all the children, move back up a level @@ -937,13 +953,17 @@ export class EndpointDocGenerator { // Otherwise, add a child and any nodes associated with it currentState.childrenCreated++; timestamp = timestamp + 1000; + const currentStateEntityID = entityIDSafeVersion(currentState.event); + const ancestry: string[] = []; + if (currentStateEntityID) { + ancestry.push(currentStateEntityID); + } + ancestry.push(...(ancestryArray(currentState.event) ?? [])); + const child = this.generateEvent({ timestamp, - parentEntityID: currentState.event.process.entity_id, - ancestry: [ - currentState.event.process.entity_id, - ...(currentState.event.process.Ext?.ancestry ?? []), - ], + parentEntityID: currentStateEntityID, + ancestry, ancestryArrayLimit: opts.ancestryArraySize, }); @@ -962,11 +982,11 @@ export class EndpointDocGenerator { processDuration = this.randomN(1000000); // This lets termination events be up to 1 million seconds after the creation event (~11 days) yield this.generateEvent({ timestamp: timestamp + processDuration * 1000, - entityID: child.process.entity_id, - parentEntityID: child.process.parent?.entity_id, + entityID: entityIDSafeVersion(child), + parentEntityID: parentEntityIDSafeVersion(child), eventCategory: ['process'], eventType: ['end'], - ancestry: child.process.Ext?.ancestry, + ancestry, ancestryArrayLimit: opts.ancestryArraySize, }); } @@ -998,7 +1018,8 @@ export class EndpointDocGenerator { ordered: boolean = false ) { let relatedEventsInfo: RelatedEventInfo[]; - let ts = node['@timestamp'] + 1; + const nodeTimestamp = timestampSafeVersion(node) ?? 0; + let ts = nodeTimestamp + 1; if (typeof relatedEvents === 'number') { relatedEventsInfo = [{ category: RelatedEventCategory.Random, count: relatedEvents }]; } else { @@ -1017,16 +1038,16 @@ export class EndpointDocGenerator { if (ordered) { ts += this.randomN(processDuration) * 1000; } else { - ts = node['@timestamp'] + this.randomN(processDuration) * 1000; + ts = nodeTimestamp + this.randomN(processDuration) * 1000; } yield this.generateEvent({ timestamp: ts, - entityID: node.process.entity_id, - parentEntityID: node.process.parent?.entity_id, + entityID: entityIDSafeVersion(node), + parentEntityID: parentEntityIDSafeVersion(node), eventCategory: eventInfo.category, eventType: eventInfo.creationType, - ancestry: node.process.Ext?.ancestry, + ancestry: ancestryArray(node), }); } } @@ -1044,12 +1065,12 @@ export class EndpointDocGenerator { alertCreationTime: number = 6 * 3600 ) { for (let i = 0; i < relatedAlerts; i++) { - const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000; + const ts = (timestampSafeVersion(node) ?? 0) + this.randomN(alertCreationTime) * 1000; yield this.generateAlert( ts, - node.process.entity_id, - node.process.parent?.entity_id, - node.process.Ext?.ancestry + entityIDSafeVersion(node), + parentEntityIDSafeVersion(node), + ancestryArray(node) ); } } diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 9a61738cd84b4..b8c2fdbe65f1e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -7,6 +7,7 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; +import { firstNonNullValue } from './models/ecs_safety_helpers'; export async function indexHostsAndAlerts( client: Client, @@ -86,7 +87,7 @@ async function indexAlerts( // eslint-disable-next-line @typescript-eslint/no-explicit-any (array: Array>, doc) => { let index = eventIndex; - if (doc.event.kind === 'alert') { + if (firstNonNullValue(doc.event?.kind) === 'alert') { index = alertIndex; } array.push({ create: { _index: index } }, doc); diff --git a/x-pack/plugins/security_solution/common/endpoint/models/ecs_safety_helpers.ts b/x-pack/plugins/security_solution/common/endpoint/models/ecs_safety_helpers.ts index 8b419e90a6ee9..5dc75bb707d0e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/ecs_safety_helpers.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/ecs_safety_helpers.ts @@ -46,12 +46,12 @@ export function values(valueOrCollection: ECSField): T[] { if (Array.isArray(valueOrCollection)) { const nonNullValues: T[] = []; for (const value of valueOrCollection) { - if (value !== null) { + if (value !== null && value !== undefined) { nonNullValues.push(value); } } return nonNullValues; - } else if (valueOrCollection !== null) { + } else if (valueOrCollection !== null && valueOrCollection !== undefined) { // if there is a single non-null value, wrap it in an array and return it. return [valueOrCollection]; } else { diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.test.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.test.ts index 6e6e0f443015b..2b0aa1601ab37 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.test.ts @@ -5,7 +5,7 @@ */ import { EndpointDocGenerator } from '../generate_data'; import { descriptiveName, isProcessRunning } from './event'; -import { ResolverEvent } from '../types'; +import { ResolverEvent, SafeResolverEvent } from '../types'; describe('Generated documents', () => { let generator: EndpointDocGenerator; @@ -17,20 +17,31 @@ describe('Generated documents', () => { it('returns the right name for a registry event', () => { const extensions = { registry: { key: `HKLM/Windows/Software/abc` } }; const event = generator.generateEvent({ eventCategory: 'registry', extensions }); - expect(descriptiveName(event)).toEqual({ subject: `HKLM/Windows/Software/abc` }); + // casting to ResolverEvent here because the `descriptiveName` function is used by the frontend is still relies + // on the unsafe ResolverEvent type. Once it's switched over to the safe version we can remove this cast. + expect(descriptiveName(event as ResolverEvent)).toEqual({ + subject: `HKLM/Windows/Software/abc`, + }); }); it('returns the right name for a network event', () => { const randomIP = `${generator.randomIP()}`; const extensions = { network: { direction: 'outbound', forwarded_ip: randomIP } }; const event = generator.generateEvent({ eventCategory: 'network', extensions }); - expect(descriptiveName(event)).toEqual({ subject: `${randomIP}`, descriptor: 'outbound' }); + // casting to ResolverEvent here because the `descriptiveName` function is used by the frontend is still relies + // on the unsafe ResolverEvent type. Once it's switched over to the safe version we can remove this cast. + expect(descriptiveName(event as ResolverEvent)).toEqual({ + subject: `${randomIP}`, + descriptor: 'outbound', + }); }); it('returns the right name for a file event', () => { const extensions = { file: { path: 'C:\\My Documents\\business\\January\\processName' } }; const event = generator.generateEvent({ eventCategory: 'file', extensions }); - expect(descriptiveName(event)).toEqual({ + // casting to ResolverEvent here because the `descriptiveName` function is used by the frontend is still relies + // on the unsafe ResolverEvent type. Once it's switched over to the safe version we can remove this cast. + expect(descriptiveName(event as ResolverEvent)).toEqual({ subject: 'C:\\My Documents\\business\\January\\processName', }); }); @@ -38,27 +49,31 @@ describe('Generated documents', () => { it('returns the right name for a dns event', () => { const extensions = { dns: { question: { name: `${generator.randomIP()}` } } }; const event = generator.generateEvent({ eventCategory: 'dns', extensions }); - expect(descriptiveName(event)).toEqual({ subject: extensions.dns.question.name }); + // casting to ResolverEvent here because the `descriptiveName` function is used by the frontend is still relies + // on the unsafe ResolverEvent type. Once it's switched over to the safe version we can remove this cast. + expect(descriptiveName(event as ResolverEvent)).toEqual({ + subject: extensions.dns.question.name, + }); }); }); describe('Process running events', () => { it('is a running event when event.type is a string', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: 'start', }); expect(isProcessRunning(event)).toBeTruthy(); }); it('is a running event when event.type is an array of strings', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: ['start'], }); expect(isProcessRunning(event)).toBeTruthy(); }); it('is a running event when event.type is an array of strings and contains start', () => { - let event: ResolverEvent = generator.generateEvent({ + let event: SafeResolverEvent = generator.generateEvent({ eventType: ['bogus', 'start', 'creation'], }); expect(isProcessRunning(event)).toBeTruthy(); @@ -70,35 +85,35 @@ describe('Generated documents', () => { }); it('is not a running event when event.type is only and end type', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: ['end'], }); expect(isProcessRunning(event)).toBeFalsy(); }); it('is not a running event when event.type is empty', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: [], }); expect(isProcessRunning(event)).toBeFalsy(); }); it('is not a running event when event.type is bogus', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: ['bogus'], }); expect(isProcessRunning(event)).toBeFalsy(); }); it('is a running event when event.type contains info', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: ['info'], }); expect(isProcessRunning(event)).toBeTruthy(); }); it('is a running event when event.type contains change', () => { - const event: ResolverEvent = generator.generateEvent({ + const event: SafeResolverEvent = generator.generateEvent({ eventType: ['bogus', 'change'], }); expect(isProcessRunning(event)).toBeTruthy(); diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index a0e9be58911c6..07208214a641a 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -9,7 +9,7 @@ import { SafeResolverEvent, SafeLegacyEndpointEvent, } from '../types'; -import { firstNonNullValue } from './ecs_safety_helpers'; +import { firstNonNullValue, hasValue, values } from './ecs_safety_helpers'; /* * Determine if a `ResolverEvent` is the legacy variety. Can be used to narrow `ResolverEvent` to `LegacyEndpointEvent`. @@ -27,32 +27,24 @@ export function isLegacyEvent(event: ResolverEvent): event is LegacyEndpointEven return (event as LegacyEndpointEvent).endgame !== undefined; } -export function isProcessRunning(event: ResolverEvent): boolean { - if (isLegacyEvent(event)) { - return ( - event.event?.type === 'process_start' || - event.event?.action === 'fork_event' || - event.event?.type === 'already_running' - ); - } - - if (Array.isArray(event.event.type)) { +export function isProcessRunning(event: SafeResolverEvent): boolean { + if (isLegacyEventSafeVersion(event)) { return ( - event.event.type.includes('start') || - event.event.type.includes('change') || - event.event.type.includes('info') + hasValue(event.event?.type, 'process_start') || + hasValue(event.event?.action, 'fork_event') || + hasValue(event.event?.type, 'already_running') ); } return ( - event.event.type === 'start' || event.event.type === 'change' || event.event.type === 'info' + hasValue(event.event?.type, 'start') || + hasValue(event.event?.type, 'change') || + hasValue(event.event?.type, 'info') ); } -export function timestampSafeVersion(event: SafeResolverEvent): string | undefined | number { - return isLegacyEventSafeVersion(event) - ? firstNonNullValue(event.endgame?.timestamp_utc) - : firstNonNullValue(event?.['@timestamp']); +export function timestampSafeVersion(event: SafeResolverEvent): undefined | number { + return firstNonNullValue(event?.['@timestamp']); } /** @@ -75,11 +67,7 @@ export function timestampAsDateSafeVersion(event: SafeResolverEvent): Date | und } export function eventTimestamp(event: ResolverEvent): string | undefined | number { - if (isLegacyEvent(event)) { - return event.endgame.timestamp_utc; - } else { - return event['@timestamp']; - } + return event['@timestamp']; } export function eventName(event: ResolverEvent): string { @@ -105,14 +93,7 @@ export function eventId(event: ResolverEvent): number | undefined | string { return event.event.id; } -export function eventSequence(event: ResolverEvent): number | undefined { - if (isLegacyEvent(event)) { - return firstNonNullValue(event.endgame.serial_event_id); - } - return firstNonNullValue(event.event?.sequence); -} - -export function eventSequenceSafeVersion(event: SafeResolverEvent): number | undefined { +export function eventSequence(event: SafeResolverEvent): number | undefined { if (isLegacyEventSafeVersion(event)) { return firstNonNullValue(event.endgame.serial_event_id); } @@ -156,16 +137,16 @@ export function parentEntityIDSafeVersion(event: SafeResolverEvent): string | un return firstNonNullValue(event.process?.parent?.entity_id); } -export function ancestryArray(event: ResolverEvent): string[] | undefined { - if (isLegacyEvent(event)) { +export function ancestryArray(event: SafeResolverEvent): string[] | undefined { + if (isLegacyEventSafeVersion(event)) { return undefined; } // this is to guard against the endpoint accidentally not sending the ancestry array // otherwise the request will fail when really we should just try using the parent entity id - return event.process.Ext?.ancestry; + return values(event.process?.Ext?.ancestry); } -export function getAncestryAsArray(event: ResolverEvent | undefined): string[] { +export function getAncestryAsArray(event: SafeResolverEvent | undefined): string[] { if (!event) { return []; } @@ -175,7 +156,7 @@ export function getAncestryAsArray(event: ResolverEvent | undefined): string[] { return ancestors; } - const parentID = parentEntityId(event); + const parentID = parentEntityIDSafeVersion(event); if (parentID) { return [parentID]; } diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index e0bd916103a28..cc40225ec1a10 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -112,6 +112,27 @@ export interface ResolverChildNode extends ResolverLifecycleNode { nextChild?: string | null; } +/** + * Safe version of `ResolverChildNode`. + */ +export interface SafeResolverChildNode extends SafeResolverLifecycleNode { + /** + * nextChild can have 3 different states: + * + * undefined: This indicates that you should not use this node for additional queries. It does not mean that node does + * not have any more direct children. The node could have more direct children but to determine that, use the + * ResolverChildren node's nextChild. + * + * null: Indicates that we have received all the children of the node. There may be more descendants though. + * + * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants + * using this node's entity_id + * + * For more information see the resolver docs on pagination [here](../../server/endpoint/routes/resolver/docs/README.md#L129) + */ + nextChild?: string | null; +} + /** * The response structure for the children route. The structure is an array of nodes where each node * has an array of lifecycle events. @@ -131,6 +152,24 @@ export interface ResolverChildren { nextChild: string | null; } +/** + * Safe version of `ResolverChildren`. + */ +export interface SafeResolverChildren { + childNodes: SafeResolverChildNode[]; + /** + * nextChild can have 2 different states: + * + * null: Indicates that we have received all the descendants that can be retrieved using this node. To retrieve more + * nodes in the tree use a cursor provided in one of the returned children. If no other cursor exists then the tree + * is complete. + * + * string: Indicates this node has more descendants that can be retrieved, pass this cursor in while using this node's + * entity_id for the request. + */ + nextChild: string | null; +} + /** * A flattened tree representing the nodes in a resolver graph. */ @@ -148,6 +187,23 @@ export interface ResolverTree { stats: ResolverNodeStats; } +/** + * Safe version of `ResolverTree`. + */ +export interface SafeResolverTree { + /** + * Origin of the tree. This is in the middle of the tree. Typically this would be the same + * process node that generated an alert. + */ + entityID: string; + children: SafeResolverChildren; + relatedEvents: Omit; + relatedAlerts: Omit; + ancestry: SafeResolverAncestry; + lifecycle: SafeResolverEvent[]; + stats: ResolverNodeStats; +} + /** * The lifecycle events (start, end etc) for a node. */ @@ -160,6 +216,18 @@ export interface ResolverLifecycleNode { stats?: ResolverNodeStats; } +/** + * Safe version of `ResolverLifecycleNode`. + */ +export interface SafeResolverLifecycleNode { + entityID: string; + lifecycle: SafeResolverEvent[]; + /** + * stats are only set when the entire tree is being fetched + */ + stats?: ResolverNodeStats; +} + /** * The response structure when searching for ancestors of a node. */ @@ -175,6 +243,21 @@ export interface ResolverAncestry { nextAncestor: string | null; } +/** + * Safe version of `ResolverAncestry`. + */ +export interface SafeResolverAncestry { + /** + * An array of ancestors with the lifecycle events grouped together + */ + ancestors: SafeResolverLifecycleNode[]; + /** + * A cursor for retrieving additional ancestors for a particular node. `null` indicates that there were no additional + * ancestors when the request returned. More could have been ingested by ES after the fact though. + */ + nextAncestor: string | null; +} + /** * Response structure for the related events route. */ @@ -198,7 +281,7 @@ export interface SafeResolverRelatedEvents { */ export interface ResolverRelatedAlerts { entityID: string; - alerts: ResolverEvent[]; + alerts: SafeResolverEvent[]; nextAlert: string | null; } @@ -251,152 +334,133 @@ export interface Host { /** * A record of hashes for something. Provides hashes in multiple formats. A favorite structure of the Elastic Endpoint. */ -interface Hashes { +type Hashes = Partial<{ /** * A hash in MD5 format. */ - md5: string; + md5: ECSField; /** * A hash in SHA-1 format. */ - sha1: string; + sha1: ECSField; /** * A hash in SHA-256 format. */ - sha256: string; -} + sha256: ECSField; +}>; -interface MalwareClassification { - identifier: string; - score: number; - threshold: number; - version: string; -} +type MalwareClassification = Partial<{ + identifier: ECSField; + score: ECSField; + threshold: ECSField; + version: ECSField; +}>; -interface ThreadFields { - id: number; - Ext: { - service_name: string; - start: number; - start_address: number; - start_address_module: string; - }; -} +type ThreadFields = Partial<{ + id: ECSField; + Ext: Partial<{ + service_name: ECSField; + start: ECSField; + start_address: ECSField; + start_address_module: ECSField; + }>; +}>; -interface DllFields { +type DllFields = Partial<{ hash: Hashes; - path: string; - pe: { - architecture: string; - }; - code_signature: { - subject_name: string; - trusted: boolean; - }; - Ext: { - compile_time: number; + path: ECSField; + pe: Partial<{ + architecture: ECSField; + }>; + code_signature: Partial<{ + subject_name: ECSField; + trusted: ECSField; + }>; + Ext: Partial<{ + compile_time: ECSField; malware_classification: MalwareClassification; - mapped_address: number; - mapped_size: number; - }; -} + mapped_address: ECSField; + mapped_size: ECSField; + }>; +}>; /** * Describes an Alert Event. */ -export interface AlertEvent { - '@timestamp': number; - agent: { - id: string; - version: string; - type: string; - }; - ecs: { - version: string; - }; - event: { - id: string; - action: string; - category: string; - kind: string; - dataset: string; - module: string; - type: string; - sequence: number; - }; - Endpoint: { - policy: { - applied: { - id: string; - status: HostPolicyResponseActionStatus; - name: string; - }; - }; - }; - process: { - command_line?: string; - pid: number; - ppid?: number; - entity_id: string; - parent?: { - pid: number; - entity_id: string; - }; - name: string; - hash: Hashes; - executable: string; - start: number; - thread?: ThreadFields[]; - uptime: number; - Ext?: { - /* - * The array has a special format. The entity_ids towards the beginning of the array are closer ancestors and the - * values towards the end of the array are more distant ancestors (grandparents). Therefore - * ancestry_array[0] == process.parent.entity_id and ancestry_array[1] == process.parent.parent.entity_id - */ - ancestry?: string[]; - code_signature: Array<{ - subject_name: string; - trusted: boolean; +export type AlertEvent = Partial<{ + event: Partial<{ + action: ECSField; + dataset: ECSField; + module: ECSField; + }>; + Endpoint: Partial<{ + policy: Partial<{ + applied: Partial<{ + id: ECSField; + status: ECSField; + name: ECSField; }>; - malware_classification?: MalwareClassification; - token: { - domain: string; - type: string; - user: string; - sid: string; - integrity_level: number; - integrity_level_name: string; - privileges?: Array<{ - description: string; - name: string; - enabled: boolean; - }>; - }; - user: string; - }; - }; - file: { - owner: string; - name: string; - path: string; - accessed: number; - mtime: number; - created: number; - size: number; - hash: Hashes; - Ext: { + }>; + }>; + process: Partial<{ + command_line: ECSField; + ppid: ECSField; + start: ECSField; + // Using ECSField as the outer because the object is expected to be an array + thread: ECSField; + uptime: ECSField; + Ext: Partial<{ + // Using ECSField as the outer because the object is expected to be an array + code_signature: ECSField< + Partial<{ + subject_name: ECSField; + trusted: ECSField; + }> + >; malware_classification: MalwareClassification; - temp_file_path: string; - code_signature: Array<{ - trusted: boolean; - subject_name: string; + token: Partial<{ + domain: ECSField; + type: ECSField; + user: ECSField; + sid: ECSField; + integrity_level: ECSField; + integrity_level_name: ECSField; + // Using ECSField as the outer because the object is expected to be an array + privileges: ECSField< + Partial<{ + description: ECSField; + name: ECSField; + enabled: ECSField; + }> + >; }>; - }; - }; - host: Host; - dll?: DllFields[]; -} + user: ECSField; + }>; + }>; + file: Partial<{ + owner: ECSField; + name: ECSField; + accessed: ECSField; + mtime: ECSField; + created: ECSField; + size: ECSField; + hash: Hashes; + Ext: Partial<{ + malware_classification: MalwareClassification; + temp_file_path: ECSField; + // Using ECSField as the outer because the object is expected to be an array + code_signature: ECSField< + Partial<{ + trusted: ECSField; + subject_name: ECSField; + }> + >; + }>; + }>; + // Using ECSField as the outer because the object is expected to be an array + dll: ECSField; +}> & + SafeEndpointEvent; /** * The status of the Endpoint Agent as reported by the Agent or the @@ -585,7 +649,7 @@ export type ResolverEvent = EndpointEvent | LegacyEndpointEvent; * All mappings in Elasticsearch support arrays. They can also return null values or be missing. For example, a `keyword` mapping could return `null` or `[null]` or `[]` or `'hi'`, or `['hi', 'there']`. We need to handle these cases in order to avoid throwing an error. * When dealing with an value that comes from ES, wrap the underlying type in `ECSField`. For example, if you have a `keyword` or `text` value coming from ES, cast it to `ECSField`. */ -export type ECSField = T | null | Array; +export type ECSField = T | null | undefined | Array; /** * A more conservative version of `ResolverEvent` that treats fields as optional and use `ECSField` to type all ECS fields. @@ -648,9 +712,7 @@ export type SafeEndpointEvent = Partial<{ subject_name: ECSField; }>; pid: ECSField; - hash: Partial<{ - md5: ECSField; - }>; + hash: Hashes; parent: Partial<{ entity_id: ECSField; name: ECSField; diff --git a/x-pack/plugins/security_solution/public/resolver/models/indexed_process_tree/__snapshots__/isometric_taxi_layout.test.ts.snap b/x-pack/plugins/security_solution/public/resolver/models/indexed_process_tree/__snapshots__/isometric_taxi_layout.test.ts.snap index db8d047c2ce86..fc0d646fd62ca 100644 --- a/x-pack/plugins/security_solution/public/resolver/models/indexed_process_tree/__snapshots__/isometric_taxi_layout.test.ts.snap +++ b/x-pack/plugins/security_solution/public/resolver/models/indexed_process_tree/__snapshots__/isometric_taxi_layout.test.ts.snap @@ -212,6 +212,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:0:1", }, "points": Array [ @@ -227,6 +231,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:0:2", }, "points": Array [ @@ -242,6 +250,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:0:8", }, "points": Array [ @@ -287,6 +299,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:1:3", }, "points": Array [ @@ -302,6 +318,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:1:4", }, "points": Array [ @@ -347,6 +367,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:2:5", }, "points": Array [ @@ -362,6 +386,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:2:6", }, "points": Array [ @@ -377,6 +405,10 @@ Object { }, Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:6:7", }, "points": Array [ @@ -584,6 +616,10 @@ Object { "edgeLineSegments": Array [ Object { "metadata": Object { + "elapsedTime": Object { + "duration": "<1", + "durationType": "millisecond", + }, "uniqueId": "edge:0:1", }, "points": Array [ diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts index e6e525334e818..1e2de06ea4af5 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts @@ -10,8 +10,9 @@ import { dataReducer } from './reducer'; import * as selectors from './selectors'; import { DataState } from '../../types'; import { DataAction } from './action'; -import { ResolverChildNode, ResolverTree } from '../../../../common/endpoint/types'; +import { ResolverChildNode, ResolverEvent, ResolverTree } from '../../../../common/endpoint/types'; import * as eventModel from '../../../../common/endpoint/models/event'; +import { values } from '../../../../common/endpoint/models/ecs_safety_helpers'; import { mockTreeFetcherParameters } from '../../mocks/tree_fetcher_parameters'; /** @@ -40,7 +41,9 @@ describe('Resolver Data Middleware', () => { // Generate a 'tree' using the Resolver generator code. This structure isn't the same as what the API returns. const baseTree = generateBaseTree(); const tree = mockResolverTree({ - events: baseTree.allEvents, + // Casting here because the generator returns the SafeResolverEvent type which isn't yet compatible with + // a lot of the frontend functions. So casting it back to the unsafe type for now. + events: baseTree.allEvents as ResolverEvent[], cursors: { childrenNextChild: 'aValidChildCursor', ancestryNextAncestor: 'aValidAncestorCursor', @@ -89,7 +92,9 @@ describe('Resolver Data Middleware', () => { type: 'serverReturnedRelatedEventData', payload: { entityID: firstChildNodeInTree.id, - events: firstChildNodeInTree.relatedEvents, + // Casting here because the generator returns the SafeResolverEvent type which isn't yet compatible with + // a lot of the frontend functions. So casting it back to the unsafe type for now. + events: firstChildNodeInTree.relatedEvents as ResolverEvent[], nextEvent: null, }, }; @@ -162,7 +167,9 @@ describe('Resolver Data Middleware', () => { type: 'serverReturnedRelatedEventData', payload: { entityID: firstChildNodeInTree.id, - events: firstChildNodeInTree.relatedEvents, + // Casting here because the generator returns the SafeResolverEvent type which isn't yet compatible with + // a lot of the frontend functions. So casting it back to the unsafe type for now. + events: firstChildNodeInTree.relatedEvents as ResolverEvent[], nextEvent: 'aValidNextEventCursor', }, }; @@ -232,7 +239,9 @@ function mockedTree() { const statsResults = compileStatsForChild(firstChildNodeInTree); const tree = mockResolverTree({ - events: baseTree.allEvents, + // Casting here because the generator returns the SafeResolverEvent type which isn't yet compatible with + // a lot of the frontend functions. So casting it back to the unsafe type for now. + events: baseTree.allEvents as ResolverEvent[], /** * Calculate children from the ResolverTree response using the children of the `Tree` we generated using the Resolver data generator code. * Compile (and attach) stats to the first child node. @@ -243,14 +252,15 @@ function mockedTree() { * related event limits should be shown. */ children: [...baseTree.children.values()].map((node: TreeNode) => { - // Treat each `TreeNode` as a `ResolverChildNode`. - // These types are almost close enough to be used interchangably (for the purposes of this test.) - const childNode: Partial = node; + const childNode: Partial = {}; + // Casting here because the generator returns the SafeResolverEvent type which isn't yet compatible with + // a lot of the frontend functions. So casting it back to the unsafe type for now. + childNode.lifecycle = node.lifecycle as ResolverEvent[]; // `TreeNode` has `id` which is the same as `entityID`. // The `ResolverChildNode` calls the entityID as `entityID`. // Set `entityID` on `childNode` since the code in test relies on it. - childNode.entityID = (childNode as TreeNode).id; + childNode.entityID = node.id; // This should only be true for the first child. if (node.id === firstChildNodeInTree.id) { @@ -315,10 +325,8 @@ function compileStatsForChild( const compiledStats = node.relatedEvents.reduce( (counts: Record, relatedEvent) => { - // `relatedEvent.event.category` is `string | string[]`. - // Wrap it in an array and flatten that array to get a `string[] | [string]` - // which we can loop over. - const categories: string[] = [relatedEvent.event.category].flat(); + // get an array of categories regardless of whether category is a string or string[] + const categories: string[] = values(relatedEvent.event?.category); for (const category of categories) { // Set the first category as 'categoryToOverCount' diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts index 54c6cf432aa89..8f68cba893108 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts @@ -5,7 +5,7 @@ */ import { SearchResponse } from 'elasticsearch'; import { esKuery } from '../../../../../../../../src/plugins/data/server'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -13,7 +13,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com /** * Builds a query for retrieving alerts for a node. */ -export class AlertsQuery extends ResolverQuery { +export class AlertsQuery extends ResolverQuery { private readonly kqlQuery: JsonObject[] = []; constructor( private readonly pagination: PaginationBuilder, @@ -68,7 +68,7 @@ export class AlertsQuery extends ResolverQuery { }; } - formatResponse(response: SearchResponse): ResolverEvent[] { + formatResponse(response: SearchResponse): SafeResolverEvent[] { return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts index 0d8a42d7a26f3..a2bdf358745c2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; import { legacyEventIndexPattern } from './legacy_event_index_pattern'; import { MSearchQuery } from './multi_searcher'; @@ -19,7 +19,7 @@ import { MSearchQuery } from './multi_searcher'; * @param R the is the type after transforming ES's response. Making this definable let's us set whether it is a resolver event * or something else. */ -export abstract class ResolverQuery implements MSearchQuery { +export abstract class ResolverQuery implements MSearchQuery { /** * * @param indexPattern the index pattern to use in the query for finding indices with documents in ES. @@ -77,7 +77,7 @@ export abstract class ResolverQuery implements MSearchQuer * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) */ async searchAndFormat(client: ILegacyScopedClusterClient, ids: string | string[]): Promise { - const res: SearchResponse = await this.search(client, ids); + const res: SearchResponse = await this.search(client, ids); return this.formatResponse(res); } @@ -113,5 +113,5 @@ export abstract class ResolverQuery implements MSearchQuer * @param response a SearchResponse from ES resulting from executing this query * @returns the translated ES response into a structured object */ - public abstract formatResponse(response: SearchResponse): T; + public abstract formatResponse(response: SearchResponse): T; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts index 6fb38a32f9581..8c7daf9451217 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { SearchResponse } from 'elasticsearch'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; import { ChildrenPaginationBuilder } from '../utils/children_pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -12,7 +12,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com /** * Builds a query for retrieving descendants of a node. */ -export class ChildrenQuery extends ResolverQuery { +export class ChildrenQuery extends ResolverQuery { constructor( private readonly pagination: ChildrenPaginationBuilder, indexPattern: string | string[], @@ -126,7 +126,7 @@ export class ChildrenQuery extends ResolverQuery { }; } - formatResponse(response: SearchResponse): ResolverEvent[] { + formatResponse(response: SearchResponse): SafeResolverEvent[] { return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts index 0969a3c360e4a..bd054d548a93a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts @@ -5,7 +5,7 @@ */ import { SearchResponse } from 'elasticsearch'; import { esKuery } from '../../../../../../../../src/plugins/data/server'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -13,7 +13,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com /** * Builds a query for retrieving related events for a node. */ -export class EventsQuery extends ResolverQuery { +export class EventsQuery extends ResolverQuery { private readonly kqlQuery: JsonObject[] = []; constructor( @@ -83,7 +83,7 @@ export class EventsQuery extends ResolverQuery { }; } - formatResponse(response: SearchResponse): ResolverEvent[] { + formatResponse(response: SearchResponse): SafeResolverEvent[] { return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts index 0b5728958e91f..ecbc5d8344928 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts @@ -6,12 +6,12 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverQuery } from './base'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; /** * Builds a query for retrieving life cycle information about a node (start, stop, etc). */ -export class LifecycleQuery extends ResolverQuery { +export class LifecycleQuery extends ResolverQuery { protected legacyQuery(endpointID: string, uniquePIDs: string[]): JsonObject { return { query: { @@ -59,7 +59,7 @@ export class LifecycleQuery extends ResolverQuery { }; } - formatResponse(response: SearchResponse): ResolverEvent[] { + formatResponse(response: SearchResponse): SafeResolverEvent[] { return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts index 02dbd92d9252b..76203973a6211 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts @@ -6,7 +6,7 @@ import { ILegacyScopedClusterClient } from 'kibana/server'; import { MSearchResponse, SearchResponse } from 'elasticsearch'; -import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** @@ -37,7 +37,7 @@ export interface QueryInfo { /** * a function to handle the response */ - handler: (response: SearchResponse) => void; + handler: (response: SearchResponse) => void; } /** @@ -65,7 +65,7 @@ export class MultiSearcher { for (const info of queries) { searchQuery.push(...info.query.buildMSearch(info.ids)); } - const res: MSearchResponse = await this.client.callAsCurrentUser('msearch', { + const res: MSearchResponse = await this.client.callAsCurrentUser('msearch', { body: searchQuery, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts index b8fa409e2ca21..50e56258b7448 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts @@ -5,7 +5,7 @@ */ import { SearchResponse } from 'elasticsearch'; import { ResolverQuery } from './base'; -import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent, EventStats } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; export interface StatsResult { @@ -185,7 +185,7 @@ export class StatsQuery extends ResolverQuery { }; } - public formatResponse(response: SearchResponse): StatsResult { + public formatResponse(response: SearchResponse): StatsResult { let alerts: Record = {}; if (response.aggregations?.alerts?.ids?.buckets) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts index efffbc10473d4..f34218ddbde9b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ResolverRelatedAlerts, ResolverEvent } from '../../../../../common/endpoint/types'; +import { ResolverRelatedAlerts, SafeResolverEvent } from '../../../../../common/endpoint/types'; import { createRelatedAlerts } from './node'; import { AlertsQuery } from '../queries/alerts'; import { PaginationBuilder } from './pagination'; @@ -45,7 +45,7 @@ export class RelatedAlertsQueryHandler implements SingleQueryHandler) => { + private handleResponse = (response: SearchResponse) => { const results = this.query.formatResponse(response); this.relatedAlerts = createRelatedAlerts( this.entityID, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts index 7dd47658bc4c1..b796913118c99 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -7,14 +7,14 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; import { - parentEntityId, - entityId, + parentEntityIDSafeVersion, + entityIDSafeVersion, getAncestryAsArray, } from '../../../../../common/endpoint/models/event'; import { - ResolverAncestry, - ResolverEvent, - ResolverLifecycleNode, + SafeResolverAncestry, + SafeResolverEvent, + SafeResolverLifecycleNode, } from '../../../../../common/endpoint/types'; import { createAncestry, createLifecycle } from './node'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -24,8 +24,8 @@ import { QueryHandler } from './fetch'; /** * Retrieve the ancestry portion of a resolver tree. */ -export class AncestryQueryHandler implements QueryHandler { - private readonly ancestry: ResolverAncestry = createAncestry(); +export class AncestryQueryHandler implements QueryHandler { + private readonly ancestry: SafeResolverAncestry = createAncestry(); private ancestorsToFind: string[]; private readonly query: LifecycleQuery; @@ -33,7 +33,7 @@ export class AncestryQueryHandler implements QueryHandler { private levels: number, indexPattern: string, legacyEndpointID: string | undefined, - originNode: ResolverLifecycleNode | undefined + originNode: SafeResolverLifecycleNode | undefined ) { this.ancestorsToFind = getAncestryAsArray(originNode?.lifecycle[0]).slice(0, levels); this.query = new LifecycleQuery(indexPattern, legacyEndpointID); @@ -41,21 +41,28 @@ export class AncestryQueryHandler implements QueryHandler { // add the origin node to the response if it exists if (originNode) { this.ancestry.ancestors.push(originNode); - this.ancestry.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; + this.ancestry.nextAncestor = parentEntityIDSafeVersion(originNode.lifecycle[0]) || null; } } - private toMapOfNodes(results: ResolverEvent[]) { - return results.reduce((nodes: Map, event: ResolverEvent) => { - const nodeId = entityId(event); - let node = nodes.get(nodeId); - if (!node) { - node = createLifecycle(nodeId, []); - } + private toMapOfNodes(results: SafeResolverEvent[]) { + return results.reduce( + (nodes: Map, event: SafeResolverEvent) => { + const nodeId = entityIDSafeVersion(event); + if (!nodeId) { + return nodes; + } + + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } - node.lifecycle.push(event); - return nodes.set(nodeId, node); - }, new Map()); + node.lifecycle.push(event); + return nodes.set(nodeId, node); + }, + new Map() + ); } private setNoMore() { @@ -64,7 +71,7 @@ export class AncestryQueryHandler implements QueryHandler { this.levels = 0; } - private handleResponse = (searchResp: SearchResponse) => { + private handleResponse = (searchResp: SearchResponse) => { const results = this.query.formatResponse(searchResp); if (results.length === 0) { this.setNoMore(); @@ -97,7 +104,7 @@ export class AncestryQueryHandler implements QueryHandler { * Hence: [D, E, B, C, A] */ this.ancestry.ancestors.push(...ancestryNodes.values()); - this.ancestry.nextAncestor = parentEntityId(results[0]) || null; + this.ancestry.nextAncestor = parentEntityIDSafeVersion(results[0]) || null; this.levels = this.levels - ancestryNodes.size; // the results come back in ascending order on timestamp so the first entry in the // results should be the further ancestor (most distant grandparent) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 78e4219aad75c..d33e9a2d70af6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -10,7 +10,7 @@ import { TreeNode, } from '../../../../../common/endpoint/generate_data'; import { ChildrenNodesHelper } from './children_helper'; -import { eventId, isProcessRunning } from '../../../../../common/endpoint/models/event'; +import { eventIDSafeVersion, isProcessRunning } from '../../../../../common/endpoint/models/event'; function getStartEvents(events: Event[]): Event[] { const startEvents: Event[] = []; @@ -179,7 +179,9 @@ describe('Children helper', () => { childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); + expect( + childrenEvents.find((child) => eventIDSafeVersion(child) === eventIDSafeVersion(event)) + ).toEqual(event); }); }); }); @@ -191,7 +193,9 @@ describe('Children helper', () => { const childrenNodes = helper.getNodes(); childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); + expect( + childrenEvents.find((child) => eventIDSafeVersion(child) === eventIDSafeVersion(event)) + ).toEqual(event); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts index b82b972b887b5..e9174548898dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts @@ -5,15 +5,15 @@ */ import { - entityId, - parentEntityId, + parentEntityIDSafeVersion, isProcessRunning, getAncestryAsArray, + entityIDSafeVersion, } from '../../../../../common/endpoint/models/event'; import { - ResolverChildNode, - ResolverEvent, - ResolverChildren, + SafeResolverChildren, + SafeResolverChildNode, + SafeResolverEvent, } from '../../../../../common/endpoint/types'; import { createChild } from './node'; import { ChildrenPaginationBuilder } from './children_pagination'; @@ -22,7 +22,7 @@ import { ChildrenPaginationBuilder } from './children_pagination'; * This class helps construct the children structure when building a resolver tree. */ export class ChildrenNodesHelper { - private readonly entityToNodeCache: Map = new Map(); + private readonly entityToNodeCache: Map = new Map(); constructor(private readonly rootID: string, private readonly limit: number) { this.entityToNodeCache.set(rootID, createChild(rootID)); @@ -31,8 +31,8 @@ export class ChildrenNodesHelper { /** * Constructs a ResolverChildren response based on the children that were previously add. */ - getNodes(): ResolverChildren { - const cacheCopy: Map = new Map(this.entityToNodeCache); + getNodes(): SafeResolverChildren { + const cacheCopy: Map = new Map(this.entityToNodeCache); const rootNode = cacheCopy.get(this.rootID); let rootNextChild = null; @@ -51,7 +51,7 @@ export class ChildrenNodesHelper { * Get the entity_ids of the nodes that are cached. */ getEntityIDs(): string[] { - const cacheCopy: Map = new Map(this.entityToNodeCache); + const cacheCopy: Map = new Map(this.entityToNodeCache); cacheCopy.delete(this.rootID); return Array.from(cacheCopy.keys()); } @@ -69,9 +69,9 @@ export class ChildrenNodesHelper { * * @param lifecycle an array of resolver lifecycle events for different process nodes returned from ES. */ - addLifecycleEvents(lifecycle: ResolverEvent[]) { + addLifecycleEvents(lifecycle: SafeResolverEvent[]) { for (const event of lifecycle) { - const entityID = entityId(event); + const entityID = entityIDSafeVersion(event); if (entityID) { const cachedChild = this.getOrCreateChildNode(entityID); cachedChild.lifecycle.push(event); @@ -86,19 +86,22 @@ export class ChildrenNodesHelper { * @param queriedNodes the entity_ids of the nodes that returned these start events * @param startEvents an array of start events returned by ES */ - addStartEvents(queriedNodes: Set, startEvents: ResolverEvent[]): Set | undefined { + addStartEvents( + queriedNodes: Set, + startEvents: SafeResolverEvent[] + ): Set | undefined { let largestAncestryArray = 0; const nodesToQueryNext: Map> = new Map(); - const nonLeafNodes: Set = new Set(); + const nonLeafNodes: Set = new Set(); - const isDistantGrandchild = (event: ResolverEvent) => { + const isDistantGrandchild = (event: SafeResolverEvent) => { const ancestry = getAncestryAsArray(event); return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]); }; for (const event of startEvents) { - const parentID = parentEntityId(event); - const entityID = entityId(event); + const parentID = parentEntityIDSafeVersion(event); + const entityID = entityIDSafeVersion(event); if (parentID && entityID && isProcessRunning(event)) { // don't actually add the start event to the node, because that'll be done in // a different call @@ -158,7 +161,7 @@ export class ChildrenNodesHelper { return nodesToQueryNext.get(largestAncestryArray); } - private setPaginationForNodes(nodes: Set, startEvents: ResolverEvent[]) { + private setPaginationForNodes(nodes: Set, startEvents: SafeResolverEvent[]) { for (const nodeEntityID of nodes.values()) { const cachedNode = this.entityToNodeCache.get(nodeEntityID); if (cachedNode) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts index ab610dc9776ca..f9f73c2ad75ff 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent, SafeResolverChildren } from '../../../../../common/endpoint/types'; import { LifecycleQuery } from '../queries/lifecycle'; import { QueryInfo } from '../queries/multi_searcher'; import { SingleQueryHandler } from './fetch'; @@ -16,8 +16,8 @@ import { createChildren } from './node'; /** * Returns the children of a resolver tree. */ -export class ChildrenLifecycleQueryHandler implements SingleQueryHandler { - private lifecycle: ResolverChildren | undefined; +export class ChildrenLifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: SafeResolverChildren | undefined; private readonly query: LifecycleQuery; constructor( private readonly childrenHelper: ChildrenNodesHelper, @@ -27,7 +27,7 @@ export class ChildrenLifecycleQueryHandler implements SingleQueryHandler) => { + private handleResponse = (response: SearchResponse) => { this.childrenHelper.addLifecycleEvents(this.query.formatResponse(response)); this.lifecycle = this.childrenHelper.getNodes(); }; @@ -50,7 +50,7 @@ export class ChildrenLifecycleQueryHandler implements SingleQueryHandler) => { + private handleResponse = (response: SearchResponse) => { const results = this.query.formatResponse(response); this.nodesToQuery = this.childrenHelper.addStartEvents(this.nodesToQuery, results) ?? new Set(); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts index 8792f917fb4d6..5c4d9a4741ad7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ResolverRelatedEvents, ResolverEvent } from '../../../../../common/endpoint/types'; +import { SafeResolverRelatedEvents, SafeResolverEvent } from '../../../../../common/endpoint/types'; import { createRelatedEvents } from './node'; import { EventsQuery } from '../queries/events'; import { PaginationBuilder } from './pagination'; @@ -28,8 +28,8 @@ export interface RelatedEventsParams { /** * This retrieves the related events for the origin node of a resolver tree. */ -export class RelatedEventsQueryHandler implements SingleQueryHandler { - private relatedEvents: ResolverRelatedEvents | undefined; +export class RelatedEventsQueryHandler implements SingleQueryHandler { + private relatedEvents: SafeResolverRelatedEvents | undefined; private readonly query: EventsQuery; private readonly limit: number; private readonly entityID: string; @@ -46,7 +46,7 @@ export class RelatedEventsQueryHandler implements SingleQueryHandler) => { + private handleResponse = (response: SearchResponse) => { const results = this.query.formatResponse(response); this.relatedEvents = createRelatedEvents( this.entityID, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 1b88f965909eb..15a9639872f2a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -6,11 +6,11 @@ import { ILegacyScopedClusterClient } from 'kibana/server'; import { - ResolverChildren, - ResolverRelatedEvents, - ResolverAncestry, + SafeResolverChildren, + SafeResolverRelatedEvents, + SafeResolverAncestry, ResolverRelatedAlerts, - ResolverLifecycleNode, + SafeResolverLifecycleNode, } from '../../../../../common/endpoint/types'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; @@ -190,7 +190,7 @@ export class Fetcher { * * @param limit upper limit of ancestors to retrieve */ - public async ancestors(limit: number): Promise { + public async ancestors(limit: number): Promise { const originNode = await this.getNode(this.id); const ancestryHandler = new AncestryQueryHandler( limit, @@ -207,7 +207,7 @@ export class Fetcher { * @param limit the number of children to retrieve for a single level * @param after a cursor to use as the starting point for retrieving children */ - public async children(limit: number, after?: string): Promise { + public async children(limit: number, after?: string): Promise { const childrenHandler = new ChildrenStartQueryHandler( limit, this.id, @@ -237,7 +237,7 @@ export class Fetcher { limit: number, after?: string, filter?: string - ): Promise { + ): Promise { const eventsHandler = new RelatedEventsQueryHandler({ limit, entityID: this.id, @@ -285,7 +285,7 @@ export class Fetcher { return tree; } - private async getNode(entityID: string): Promise { + private async getNode(entityID: string): Promise { const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); const results = await query.searchAndFormat(this.client, entityID); if (results.length === 0) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts index ab0501e099490..d4dc12d5e8b66 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts @@ -6,7 +6,7 @@ import { SearchResponse } from 'elasticsearch'; import { ILegacyScopedClusterClient } from 'kibana/server'; -import { ResolverEvent, ResolverLifecycleNode } from '../../../../../common/endpoint/types'; +import { SafeResolverEvent, SafeResolverLifecycleNode } from '../../../../../common/endpoint/types'; import { LifecycleQuery } from '../queries/lifecycle'; import { QueryInfo } from '../queries/multi_searcher'; import { SingleQueryHandler } from './fetch'; @@ -15,8 +15,8 @@ import { createLifecycle } from './node'; /** * Retrieve the lifecycle events for a node. */ -export class LifecycleQueryHandler implements SingleQueryHandler { - private lifecycle: ResolverLifecycleNode | undefined; +export class LifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: SafeResolverLifecycleNode | undefined; private readonly query: LifecycleQuery; constructor( private readonly entityID: string, @@ -26,7 +26,7 @@ export class LifecycleQueryHandler implements SingleQueryHandler) => { + private handleResponse = (response: SearchResponse) => { const results = this.query.formatResponse(response); if (results.length !== 0) { this.lifecycle = createLifecycle(this.entityID, results); @@ -51,7 +51,7 @@ export class LifecycleQueryHandler implements SingleQueryHandler { const generator = new EndpointDocGenerator(); - const getSearchAfterInfo = (events: EndpointEvent[]) => { + const getSearchAfterInfo = (events: SafeEndpointEvent[]) => { const lastEvent = events[events.length - 1]; - return [lastEvent['@timestamp'], lastEvent.event.id]; + return [timestampSafeVersion(lastEvent), eventIDSafeVersion(lastEvent)]; }; describe('cursor', () => { const root = generator.generateEvent(); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts index 4a6c65e55a6b6..af0311a262f30 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts @@ -4,8 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ResolverEvent } from '../../../../../common/endpoint/types'; -import { eventId } from '../../../../../common/endpoint/models/event'; +import { SafeResolverEvent } from '../../../../../common/endpoint/types'; +import { + eventIDSafeVersion, + timestampSafeVersion, +} from '../../../../../common/endpoint/models/event'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; import { ChildrenPaginationCursor } from './children_pagination'; @@ -116,11 +119,12 @@ export class PaginationBuilder { * * @param results the events that were returned by the ES query */ - static buildCursor(results: ResolverEvent[]): string | null { + static buildCursor(results: SafeResolverEvent[]): string | null { const lastResult = results[results.length - 1]; const cursor = { - timestamp: lastResult['@timestamp'], - eventID: eventId(lastResult) === undefined ? '' : String(eventId(lastResult)), + timestamp: timestampSafeVersion(lastResult) ?? 0, + eventID: + eventIDSafeVersion(lastResult) === undefined ? '' : String(eventIDSafeVersion(lastResult)), }; return urlEncodeCursor(cursor); } @@ -131,7 +135,10 @@ export class PaginationBuilder { * @param requestLimit the request limit for a query. * @param results the events that were returned by the ES query */ - static buildCursorRequestLimit(requestLimit: number, results: ResolverEvent[]): string | null { + static buildCursorRequestLimit( + requestLimit: number, + results: SafeResolverEvent[] + ): string | null { if (requestLimit <= results.length && results.length > 0) { return PaginationBuilder.buildCursor(results); } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts index 21db11f3affd3..290af87a61b1d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.test.ts @@ -7,28 +7,28 @@ import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; import { Tree } from './tree'; import { - ResolverAncestry, - ResolverEvent, - ResolverRelatedEvents, + SafeResolverAncestry, + SafeResolverEvent, + SafeResolverRelatedEvents, } from '../../../../../common/endpoint/types'; -import { entityId } from '../../../../../common/endpoint/models/event'; +import { entityIDSafeVersion } from '../../../../../common/endpoint/models/event'; describe('Tree', () => { const generator = new EndpointDocGenerator(); describe('ancestry', () => { // transform the generator's array of events into the format expected by the tree class - const ancestorInfo: ResolverAncestry = { + const ancestorInfo: SafeResolverAncestry = { ancestors: generator .createAlertEventAncestry({ ancestors: 5, percentTerminated: 0, percentWithRelated: 0 }) .filter((event) => { - return event.event.kind === 'event'; + return event.event?.kind === 'event'; }) .map((event) => { return { - entityID: event.process.entity_id, + entityID: entityIDSafeVersion(event) ?? '', // The generator returns Events, but the tree needs a ResolverEvent - lifecycle: [event as ResolverEvent], + lifecycle: [event as SafeResolverEvent], }; }), nextAncestor: 'hello', @@ -39,7 +39,7 @@ describe('Tree', () => { const ids = tree.ids(); ids.forEach((id) => { const foundAncestor = ancestorInfo.ancestors.find( - (ancestor) => entityId(ancestor.lifecycle[0]) === id + (ancestor) => entityIDSafeVersion(ancestor.lifecycle[0]) === id ); expect(foundAncestor).not.toBeUndefined(); }); @@ -50,12 +50,12 @@ describe('Tree', () => { describe('related events', () => { it('adds related events to the tree', () => { const root = generator.generateEvent(); - const events: ResolverRelatedEvents = { - entityID: root.process.entity_id, + const events: SafeResolverRelatedEvents = { + entityID: entityIDSafeVersion(root) ?? '', events: Array.from(generator.relatedEventsGenerator(root)), nextEvent: null, }; - const tree = new Tree(root.process.entity_id, { relatedEvents: events }); + const tree = new Tree(entityIDSafeVersion(root) ?? '', { relatedEvents: events }); const rendered = tree.render(); expect(rendered.relatedEvents.nextEvent).toBeNull(); expect(rendered.relatedEvents.events).toStrictEqual(events.events); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts index 3f941851a4143..dd493d70ffcd3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/tree.ts @@ -6,26 +6,26 @@ import _ from 'lodash'; import { - ResolverEvent, + SafeResolverEvent, ResolverNodeStats, - ResolverRelatedEvents, - ResolverAncestry, - ResolverTree, - ResolverChildren, + SafeResolverRelatedEvents, + SafeResolverAncestry, + SafeResolverTree, + SafeResolverChildren, ResolverRelatedAlerts, } from '../../../../../common/endpoint/types'; import { createTree } from './node'; interface Node { entityID: string; - lifecycle: ResolverEvent[]; + lifecycle: SafeResolverEvent[]; stats?: ResolverNodeStats; } export interface Options { - relatedEvents?: ResolverRelatedEvents; - ancestry?: ResolverAncestry; - children?: ResolverChildren; + relatedEvents?: SafeResolverRelatedEvents; + ancestry?: SafeResolverAncestry; + children?: SafeResolverChildren; relatedAlerts?: ResolverRelatedAlerts; } @@ -37,7 +37,7 @@ export interface Options { */ export class Tree { protected cache: Map = new Map(); - protected tree: ResolverTree; + protected tree: SafeResolverTree; constructor(protected readonly id: string, options: Options = {}) { const tree = createTree(this.id); @@ -55,7 +55,7 @@ export class Tree { * * @returns the origin ResolverNode */ - public render(): ResolverTree { + public render(): SafeResolverTree { return this.tree; } @@ -73,7 +73,7 @@ export class Tree { * * @param relatedEventsInfo is the related events and pagination information to add to the tree. */ - private addRelatedEvents(relatedEventsInfo: ResolverRelatedEvents | undefined) { + private addRelatedEvents(relatedEventsInfo: SafeResolverRelatedEvents | undefined) { if (!relatedEventsInfo) { return; } @@ -101,7 +101,7 @@ export class Tree { * * @param ancestorInfo is the ancestors and pagination information to add to the tree. */ - private addAncestors(ancestorInfo: ResolverAncestry | undefined) { + private addAncestors(ancestorInfo: SafeResolverAncestry | undefined) { if (!ancestorInfo) { return; } @@ -132,7 +132,7 @@ export class Tree { } } - private addChildren(children: ResolverChildren | undefined) { + private addChildren(children: SafeResolverChildren | undefined) { if (!children) { return; } diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/alerts.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/alerts.ts index 82d844aae8016..bf7ed711b75a5 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/alerts.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/alerts.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { eventId } from '../../../../plugins/security_solution/common/endpoint/models/event'; +import { + eventIDSafeVersion, + timestampSafeVersion, +} from '../../../../plugins/security_solution/common/endpoint/models/event'; import { ResolverRelatedAlerts } from '../../../../plugins/security_solution/common/endpoint/types'; import { FtrProviderContext } from '../../ftr_provider_context'; import { @@ -69,7 +72,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should allow alerts to be filtered', async () => { - const filter = `not event.id:"${tree.origin.relatedAlerts[0].event.id}"`; + const filter = `not event.id:"${tree.origin.relatedAlerts[0].event?.id}"`; const { body }: { body: ResolverRelatedAlerts } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/alerts`) .set('kbn-xsrf', 'xxx') @@ -84,7 +87,7 @@ export default function ({ getService }: FtrProviderContext) { // should not find the alert that we excluded in the filter expect( body.alerts.find((bodyAlert) => { - return eventId(bodyAlert) === tree.origin.relatedAlerts[0].event.id; + return eventIDSafeVersion(bodyAlert) === tree.origin.relatedAlerts[0].event?.id; }) ).to.not.be.ok(); }); @@ -135,14 +138,16 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); const sortedAsc = [...tree.origin.relatedAlerts].sort((event1, event2) => { // this sorts the events by timestamp in ascending order - const diff = event1['@timestamp'] - event2['@timestamp']; + const diff = (timestampSafeVersion(event1) ?? 0) - (timestampSafeVersion(event2) ?? 0); + const event1ID = eventIDSafeVersion(event1) ?? 0; + const event2ID = eventIDSafeVersion(event2) ?? 0; // if the timestamps are the same, fallback to the event.id sorted in // ascending order if (diff === 0) { - if (event1.event.id < event2.event.id) { + if (event1ID < event2ID) { return -1; } - if (event1.event.id > event2.event.id) { + if (event1ID > event2ID) { return 1; } return 0; @@ -152,7 +157,7 @@ export default function ({ getService }: FtrProviderContext) { expect(body.alerts.length).to.eql(4); for (let i = 0; i < body.alerts.length; i++) { - expect(eventId(body.alerts[i])).to.equal(sortedAsc[i].event.id); + expect(eventIDSafeVersion(body.alerts[i])).to.equal(sortedAsc[i].event?.id); } }); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/children.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/children.ts index 2dec3c755a93b..49e24ff67fa77 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/children.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/children.ts @@ -5,14 +5,17 @@ */ import expect from '@kbn/expect'; import { SearchResponse } from 'elasticsearch'; -import { entityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; +import { + entityIDSafeVersion, + timestampSafeVersion, +} from '../../../../plugins/security_solution/common/endpoint/models/event'; import { eventsIndexPattern } from '../../../../plugins/security_solution/common/endpoint/constants'; import { ChildrenPaginationBuilder } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/utils/children_pagination'; import { ChildrenQuery } from '../../../../plugins/security_solution/server/endpoint/routes/resolver/queries/children'; import { - ResolverTree, - ResolverEvent, - ResolverChildren, + SafeResolverTree, + SafeResolverEvent, + SafeResolverChildren, } from '../../../../plugins/security_solution/common/endpoint/types'; import { FtrProviderContext } from '../../ftr_provider_context'; import { @@ -20,6 +23,7 @@ import { EndpointDocGenerator, } from '../../../../plugins/security_solution/common/endpoint/generate_data'; import { InsertedEvents } from '../../services/resolver'; +import { createAncestryArray } from './common'; export default function resolverAPIIntegrationTests({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -40,20 +44,20 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC // Origin -> infoEvent -> startEvent -> execEvent origin = generator.generateEvent(); infoEvent = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), eventType: ['info'], }); startEvent = generator.generateEvent({ - parentEntityID: infoEvent.process.entity_id, - ancestry: [infoEvent.process.entity_id, origin.process.entity_id], + parentEntityID: entityIDSafeVersion(infoEvent), + ancestry: createAncestryArray([infoEvent, origin]), eventType: ['start'], }); execEvent = generator.generateEvent({ - parentEntityID: startEvent.process.entity_id, - ancestry: [startEvent.process.entity_id, infoEvent.process.entity_id], + parentEntityID: entityIDSafeVersion(startEvent), + ancestry: createAncestryArray([startEvent, infoEvent]), eventType: ['change'], }); genData = await resolver.insertEvents([origin, infoEvent, startEvent, execEvent]); @@ -64,13 +68,13 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC }); it('finds all the children of the origin', async () => { - const { body }: { body: ResolverTree } = await supertest - .get(`/api/endpoint/resolver/${origin.process.entity_id}?children=100`) + const { body }: { body: SafeResolverTree } = await supertest + .get(`/api/endpoint/resolver/${origin.process?.entity_id}?children=100`) .expect(200); expect(body.children.childNodes.length).to.be(3); - expect(body.children.childNodes[0].entityID).to.be(infoEvent.process.entity_id); - expect(body.children.childNodes[1].entityID).to.be(startEvent.process.entity_id); - expect(body.children.childNodes[2].entityID).to.be(execEvent.process.entity_id); + expect(body.children.childNodes[0].entityID).to.be(infoEvent.process?.entity_id); + expect(body.children.childNodes[1].entityID).to.be(startEvent.process?.entity_id); + expect(body.children.childNodes[2].entityID).to.be(execEvent.process?.entity_id); }); }); @@ -86,23 +90,23 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC // Origin -> (infoEvent, startEvent, execEvent are all for the same node) origin = generator.generateEvent(); startEvent = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), eventType: ['start'], }); infoEvent = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], - entityID: startEvent.process.entity_id, + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), + entityID: entityIDSafeVersion(startEvent), eventType: ['info'], }); execEvent = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), eventType: ['change'], - entityID: startEvent.process.entity_id, + entityID: entityIDSafeVersion(startEvent), }); genData = await resolver.insertEvents([origin, infoEvent, startEvent, execEvent]); }); @@ -117,12 +121,12 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC eventsIndexPattern ); // [1] here gets the body portion of the array - const [, query] = childrenQuery.buildMSearch(origin.process.entity_id); - const { body } = await es.search>({ body: query }); + const [, query] = childrenQuery.buildMSearch(entityIDSafeVersion(origin) ?? ''); + const { body } = await es.search>({ body: query }); expect(body.hits.hits.length).to.be(1); const event = body.hits.hits[0]._source; - expect(entityId(event)).to.be(startEvent.process.entity_id); + expect(entityIDSafeVersion(event)).to.be(startEvent.process?.entity_id); expect(event.event?.type).to.eql(['start']); }); }); @@ -139,25 +143,25 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC // Origin -> (infoEvent, startEvent, execEvent are all for the same node) origin = generator.generateEvent(); startEvent = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), eventType: ['start'], }); infoEvent = generator.generateEvent({ - timestamp: startEvent['@timestamp'] + 100, - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], - entityID: startEvent.process.entity_id, + timestamp: (timestampSafeVersion(startEvent) ?? 0) + 100, + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), + entityID: entityIDSafeVersion(startEvent), eventType: ['info'], }); execEvent = generator.generateEvent({ - timestamp: infoEvent['@timestamp'] + 100, - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + timestamp: (timestampSafeVersion(infoEvent) ?? 0) + 100, + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), eventType: ['change'], - entityID: startEvent.process.entity_id, + entityID: entityIDSafeVersion(startEvent), }); genData = await resolver.insertEvents([origin, infoEvent, startEvent, execEvent]); }); @@ -167,37 +171,37 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC }); it('retrieves the same node three times', async () => { - let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${origin.process.entity_id}/children?children=1`) + let { body }: { body: SafeResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${origin.process?.entity_id}/children?children=1`) .expect(200); expect(body.childNodes.length).to.be(1); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].entityID).to.be(startEvent.process.entity_id); - expect(body.childNodes[0].lifecycle[0].event?.type).to.eql(startEvent.event.type); + expect(body.childNodes[0].entityID).to.be(startEvent.process?.entity_id); + expect(body.childNodes[0].lifecycle[0].event?.type).to.eql(startEvent.event?.type); ({ body } = await supertest .get( - `/api/endpoint/resolver/${origin.process.entity_id}/children?children=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${origin.process?.entity_id}/children?children=1&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.be(1); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].entityID).to.be(infoEvent.process.entity_id); - expect(body.childNodes[0].lifecycle[1].event?.type).to.eql(infoEvent.event.type); + expect(body.childNodes[0].entityID).to.be(infoEvent.process?.entity_id); + expect(body.childNodes[0].lifecycle[1].event?.type).to.eql(infoEvent.event?.type); ({ body } = await supertest .get( - `/api/endpoint/resolver/${origin.process.entity_id}/children?children=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${origin.process?.entity_id}/children?children=1&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.be(1); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].entityID).to.be(infoEvent.process.entity_id); - expect(body.childNodes[0].lifecycle[2].event?.type).to.eql(execEvent.event.type); + expect(body.childNodes[0].entityID).to.be(infoEvent.process?.entity_id); + expect(body.childNodes[0].lifecycle[2].event?.type).to.eql(execEvent.event?.type); ({ body } = await supertest .get( - `/api/endpoint/resolver/${origin.process.entity_id}/children?children=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${origin.process?.entity_id}/children?children=1&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.be(0); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts index 92d14fb94a2d8..2c59863099ae7 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/common.ts @@ -6,14 +6,15 @@ import _ from 'lodash'; import expect from '@kbn/expect'; import { - ResolverChildNode, - ResolverLifecycleNode, - ResolverEvent, + SafeResolverChildNode, + SafeResolverLifecycleNode, + SafeResolverEvent, ResolverNodeStats, } from '../../../../plugins/security_solution/common/endpoint/types'; import { - parentEntityId, - eventId, + parentEntityIDSafeVersion, + entityIDSafeVersion, + eventIDSafeVersion, } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { Event, @@ -23,13 +24,33 @@ import { categoryMapping, } from '../../../../plugins/security_solution/common/endpoint/generate_data'; +/** + * Creates the ancestry array based on an array of events. The order of the ancestry array will match the order + * of the events passed in. + * + * @param events an array of generated events + */ +export const createAncestryArray = (events: Event[]) => { + const ancestry: string[] = []; + for (const event of events) { + const entityID = entityIDSafeVersion(event); + if (entityID) { + ancestry.push(entityID); + } + } + return ancestry; +}; + /** * Check that the given lifecycle is in the resolver tree's corresponding map * * @param node a lifecycle node containing the start and end events for a node * @param nodeMap a map of entity_ids to nodes to look for the passed in `node` */ -const expectLifecycleNodeInMap = (node: ResolverLifecycleNode, nodeMap: Map) => { +const expectLifecycleNodeInMap = ( + node: SafeResolverLifecycleNode, + nodeMap: Map +) => { const genNode = nodeMap.get(node.entityID); expect(genNode).to.be.ok(); compareArrays(genNode!.lifecycle, node.lifecycle, true); @@ -44,7 +65,7 @@ const expectLifecycleNodeInMap = (node: ResolverLifecycleNode, nodeMap: Map { @@ -52,7 +73,7 @@ export const verifyAncestry = ( const groupedAncestors = _.groupBy(ancestors, (ancestor) => ancestor.entityID); // group by parent entity_id const groupedAncestorsParent = _.groupBy(ancestors, (ancestor) => - parentEntityId(ancestor.lifecycle[0]) + parentEntityIDSafeVersion(ancestor.lifecycle[0]) ); // make sure there aren't any nodes with the same entity_id expect(Object.keys(groupedAncestors).length).to.eql(ancestors.length); @@ -69,7 +90,7 @@ export const verifyAncestry = ( let foundParents = 0; let node = ancestors[0]; for (let i = 0; i < ancestors.length; i++) { - const parentID = parentEntityId(node.lifecycle[0]); + const parentID = parentEntityIDSafeVersion(node.lifecycle[0]); if (parentID !== undefined) { const nextNode = groupedAncestors[parentID]; if (!nextNode) { @@ -95,12 +116,12 @@ export const verifyAncestry = ( * * @param ancestors an array of ancestor nodes */ -export const retrieveDistantAncestor = (ancestors: ResolverLifecycleNode[]) => { +export const retrieveDistantAncestor = (ancestors: SafeResolverLifecycleNode[]) => { // group the ancestors by their entity_id mapped to a lifecycle node const groupedAncestors = _.groupBy(ancestors, (ancestor) => ancestor.entityID); let node = ancestors[0]; for (let i = 0; i < ancestors.length; i++) { - const parentID = parentEntityId(node.lifecycle[0]); + const parentID = parentEntityIDSafeVersion(node.lifecycle[0]); if (parentID !== undefined) { const nextNode = groupedAncestors[parentID]; if (nextNode) { @@ -122,7 +143,7 @@ export const retrieveDistantAncestor = (ancestors: ResolverLifecycleNode[]) => { * @param childrenPerParent an optional number to compare that there are a certain number of children for each parent */ export const verifyChildren = ( - children: ResolverChildNode[], + children: SafeResolverChildNode[], tree: Tree, numberOfParents?: number, childrenPerParent?: number @@ -132,7 +153,9 @@ export const verifyChildren = ( // make sure each child is unique expect(Object.keys(groupedChildren).length).to.eql(children.length); if (numberOfParents !== undefined) { - const groupParent = _.groupBy(children, (child) => parentEntityId(child.lifecycle[0])); + const groupParent = _.groupBy(children, (child) => + parentEntityIDSafeVersion(child.lifecycle[0]) + ); expect(Object.keys(groupParent).length).to.eql(numberOfParents); if (childrenPerParent !== undefined) { Object.values(groupParent).forEach((childNodes) => @@ -155,7 +178,7 @@ export const verifyChildren = ( */ export const compareArrays = ( expected: Event[], - toTest: ResolverEvent[], + toTest: SafeResolverEvent[], lengthCheck: boolean = false ) => { if (lengthCheck) { @@ -168,7 +191,7 @@ export const compareArrays = ( // we're only checking that the event ids are the same here. The reason we can't check the entire document // is because ingest pipelines are used to add fields to the document when it is received by elasticsearch, // therefore it will not be the same as the document created by the generator - return eventId(toTestEvent) === eventId(arrEvent); + return eventIDSafeVersion(toTestEvent) === eventIDSafeVersion(arrEvent); }) ).to.be.ok(); }); @@ -212,7 +235,7 @@ export const verifyStats = ( * @param categories the related event info used when generating the resolver tree */ export const verifyLifecycleStats = ( - nodes: ResolverLifecycleNode[], + nodes: SafeResolverLifecycleNode[], categories: RelatedEventInfo[], relatedAlerts: number ) => { diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts index cb6c49e17c712..e6d5e8fccd00d 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/entity_id.ts @@ -4,9 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; +import { entityIDSafeVersion } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { eventsIndexPattern } from '../../../../plugins/security_solution/common/endpoint/constants'; import { - ResolverTree, + SafeResolverTree, ResolverEntityIndex, } from '../../../../plugins/security_solution/common/endpoint/types'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -15,19 +16,26 @@ import { Event, } from '../../../../plugins/security_solution/common/endpoint/generate_data'; import { InsertedEvents } from '../../services/resolver'; +import { createAncestryArray } from './common'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const resolver = getService('resolverGenerator'); const generator = new EndpointDocGenerator('resolver'); + const setEntityIDEmptyString = (event: Event) => { + if (event.process?.entity_id) { + event.process.entity_id = ''; + } + }; + describe('Resolver handling of entity ids', () => { describe('entity api', () => { let origin: Event; let genData: InsertedEvents; before(async () => { origin = generator.generateEvent({ parentEntityID: 'a' }); - origin.process.entity_id = ''; + setEntityIDEmptyString(origin); genData = await resolver.insertEvents([origin]); }); @@ -57,16 +65,16 @@ export default function ({ getService }: FtrProviderContext) { // should not be returned by the backend. origin = generator.generateEvent({ entityID: 'a' }); childNoEntityID = generator.generateEvent({ - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), }); // force it to be empty - childNoEntityID.process.entity_id = ''; + setEntityIDEmptyString(childNoEntityID); childWithEntityID = generator.generateEvent({ entityID: 'b', - parentEntityID: origin.process.entity_id, - ancestry: [origin.process.entity_id], + parentEntityID: entityIDSafeVersion(origin), + ancestry: createAncestryArray([origin]), }); events = [origin, childNoEntityID, childWithEntityID]; genData = await resolver.insertEvents(events); @@ -77,11 +85,11 @@ export default function ({ getService }: FtrProviderContext) { }); it('does not find children without a process entity_id', async () => { - const { body }: { body: ResolverTree } = await supertest - .get(`/api/endpoint/resolver/${origin.process.entity_id}`) + const { body }: { body: SafeResolverTree } = await supertest + .get(`/api/endpoint/resolver/${origin.process?.entity_id}`) .expect(200); expect(body.children.childNodes.length).to.be(1); - expect(body.children.childNodes[0].entityID).to.be(childWithEntityID.process.entity_id); + expect(body.children.childNodes[0].entityID).to.be(childWithEntityID.process?.entity_id); }); }); @@ -101,21 +109,21 @@ export default function ({ getService }: FtrProviderContext) { }); ancestor1 = generator.generateEvent({ entityID: '1', - parentEntityID: ancestor2.process.entity_id, - ancestry: [ancestor2.process.entity_id], + parentEntityID: entityIDSafeVersion(ancestor2), + ancestry: createAncestryArray([ancestor2]), }); // we'll insert an event that doesn't have an entity id so if the backend does search for it, it should be // returned and our test should fail ancestorNoEntityID = generator.generateEvent({ - ancestry: [ancestor2.process.entity_id], + ancestry: createAncestryArray([ancestor2]), }); - ancestorNoEntityID.process.entity_id = ''; + setEntityIDEmptyString(ancestorNoEntityID); origin = generator.generateEvent({ entityID: 'a', - parentEntityID: ancestor1.process.entity_id, - ancestry: ['', ancestor2.process.entity_id], + parentEntityID: entityIDSafeVersion(ancestor1), + ancestry: ['', ...createAncestryArray([ancestor2])], }); events = [origin, ancestor1, ancestor2, ancestorNoEntityID]; @@ -127,11 +135,11 @@ export default function ({ getService }: FtrProviderContext) { }); it('does not query for ancestors that have an empty string for the entity_id', async () => { - const { body }: { body: ResolverTree } = await supertest - .get(`/api/endpoint/resolver/${origin.process.entity_id}`) + const { body }: { body: SafeResolverTree } = await supertest + .get(`/api/endpoint/resolver/${origin.process?.entity_id}`) .expect(200); expect(body.ancestry.ancestors.length).to.be(1); - expect(body.ancestry.ancestors[0].entityID).to.be(ancestor2.process.entity_id); + expect(body.ancestry.ancestors[0].entityID).to.be(ancestor2.process?.entity_id); }); }); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts index c0e4e466c7b62..4e248f52ec297 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/events.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import expect from '@kbn/expect'; -import { eventId } from '../../../../plugins/security_solution/common/endpoint/models/event'; -import { ResolverRelatedEvents } from '../../../../plugins/security_solution/common/endpoint/types'; +import { eventIDSafeVersion } from '../../../../plugins/security_solution/common/endpoint/models/event'; +import { SafeResolverRelatedEvents } from '../../../../plugins/security_solution/common/endpoint/types'; import { FtrProviderContext } from '../../ftr_provider_context'; import { Tree, @@ -59,7 +59,7 @@ export default function ({ getService }: FtrProviderContext) { const cursor = 'eyJ0aW1lc3RhbXAiOjE1ODE0NTYyNTUwMDAsImV2ZW50SUQiOiI5NDA0MyJ9'; it('should return details for the root node', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -69,7 +69,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('returns no values when there is no more data', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest // after is set to the document id of the last event so there shouldn't be any more after it .post( `/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}&afterEvent=${cursor}` @@ -82,7 +82,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return the first page of information when the cursor is invalid', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post( `/api/endpoint/resolver/${entityID}/events?legacyEndpointID=${endpointID}&afterEvent=blah` ) @@ -93,7 +93,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return no results for an invalid endpoint ID', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${entityID}/events?legacyEndpointID=foo`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -120,7 +120,7 @@ export default function ({ getService }: FtrProviderContext) { describe('endpoint events', () => { it('should not find any events', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/5555/events`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -129,7 +129,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return details for the root node', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/events`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -140,7 +140,7 @@ export default function ({ getService }: FtrProviderContext) { it('should allow for the events to be filtered', async () => { const filter = `event.category:"${RelatedEventCategory.Driver}"`; - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/events`) .set('kbn-xsrf', 'xxx') .send({ @@ -156,7 +156,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return paginated results for the root node', async () => { - let { body }: { body: ResolverRelatedEvents } = await supertest + let { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/events?events=2`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -185,7 +185,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return the first page of information when the cursor is invalid', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/events?afterEvent=blah`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -195,7 +195,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should sort the events in descending order', async () => { - const { body }: { body: ResolverRelatedEvents } = await supertest + const { body }: { body: SafeResolverRelatedEvents } = await supertest .post(`/api/endpoint/resolver/${tree.origin.id}/events`) .set('kbn-xsrf', 'xxx') .expect(200); @@ -204,8 +204,8 @@ export default function ({ getService }: FtrProviderContext) { // the last element in the array so let's reverse it const relatedEvents = tree.origin.relatedEvents.reverse(); for (let i = 0; i < body.events.length; i++) { - expect(body.events[i].event?.category).to.equal(relatedEvents[i].event.category); - expect(eventId(body.events[i])).to.equal(relatedEvents[i].event.id); + expect(body.events[i].event?.category).to.equal(relatedEvents[i].event?.category); + expect(eventIDSafeVersion(body.events[i])).to.equal(relatedEvents[i].event?.id); } }); }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts index 957d559087f5e..837af6a940f5c 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/resolver/tree.ts @@ -5,12 +5,12 @@ */ import expect from '@kbn/expect'; import { - ResolverAncestry, - ResolverChildren, - ResolverTree, - LegacyEndpointEvent, + SafeResolverAncestry, + SafeResolverChildren, + SafeResolverTree, + SafeLegacyEndpointEvent, } from '../../../../plugins/security_solution/common/endpoint/types'; -import { parentEntityId } from '../../../../plugins/security_solution/common/endpoint/models/event'; +import { parentEntityIDSafeVersion } from '../../../../plugins/security_solution/common/endpoint/models/event'; import { FtrProviderContext } from '../../ftr_provider_context'; import { Tree, @@ -71,7 +71,7 @@ export default function ({ getService }: FtrProviderContext) { const entityID = '94042'; it('should return details for the root node', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get( `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=5` ) @@ -82,7 +82,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should have a populated next parameter', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get( `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` ) @@ -91,7 +91,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should handle an ancestors param request', async () => { - let { body }: { body: ResolverAncestry } = await supertest + let { body }: { body: SafeResolverAncestry } = await supertest .get( `/api/endpoint/resolver/${entityID}/ancestry?legacyEndpointID=${endpointID}&ancestors=0` ) @@ -110,14 +110,14 @@ export default function ({ getService }: FtrProviderContext) { describe('endpoint events', () => { it('should return the origin node at the front of the array', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) .expect(200); expect(body.ancestors[0].entityID).to.eql(tree.origin.id); }); it('should return details for the root node', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=9`) .expect(200); // the tree we generated had 5 ancestors + 1 origin node @@ -128,7 +128,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('should handle an invalid id', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get(`/api/endpoint/resolver/alskdjflasj/ancestry`) .expect(200); expect(body.ancestors).to.be.empty(); @@ -136,18 +136,20 @@ export default function ({ getService }: FtrProviderContext) { }); it('should have a populated next parameter', async () => { - const { body }: { body: ResolverAncestry } = await supertest + const { body }: { body: SafeResolverAncestry } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=2`) .expect(200); // it should have 2 ancestors + 1 origin expect(body.ancestors.length).to.eql(3); verifyAncestry(body.ancestors, tree, false); const distantGrandparent = retrieveDistantAncestor(body.ancestors); - expect(body.nextAncestor).to.eql(parentEntityId(distantGrandparent.lifecycle[0])); + expect(body.nextAncestor).to.eql( + parentEntityIDSafeVersion(distantGrandparent.lifecycle[0]) + ); }); it('should handle multiple ancestor requests', async () => { - let { body }: { body: ResolverAncestry } = await supertest + let { body }: { body: SafeResolverAncestry } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/ancestry?ancestors=3`) .expect(200); expect(body.ancestors.length).to.eql(4); @@ -171,7 +173,7 @@ export default function ({ getService }: FtrProviderContext) { const entityID = '94041'; it('returns child process lifecycle events', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}`) .expect(200); expect(body.childNodes.length).to.eql(1); @@ -179,12 +181,12 @@ export default function ({ getService }: FtrProviderContext) { expect( // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent // here, so to avoid it complaining we'll just force it - (body.childNodes[0].lifecycle[0] as LegacyEndpointEvent).endgame.unique_pid + (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid ).to.eql(94042); }); it('returns multiple levels of child process lifecycle events', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) .expect(200); expect(body.childNodes.length).to.eql(10); @@ -193,12 +195,12 @@ export default function ({ getService }: FtrProviderContext) { expect( // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent // here, so to avoid it complaining we'll just force it - (body.childNodes[0].lifecycle[0] as LegacyEndpointEvent).endgame.unique_pid + (body.childNodes[0].lifecycle[0] as SafeLegacyEndpointEvent).endgame.unique_pid ).to.eql(93932); }); it('returns no values when there is no more data', async () => { - let { body }: { body: ResolverChildren } = await supertest + let { body }: { body: SafeResolverChildren } = await supertest .get( // there should only be a single child for this node `/api/endpoint/resolver/94041/children?legacyEndpointID=${endpointID}&children=1` @@ -216,7 +218,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('returns the first page of information when the cursor is invalid', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get( `/api/endpoint/resolver/${entityID}/children?legacyEndpointID=${endpointID}&afterChild=blah` ) @@ -236,7 +238,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('returns empty events without a matching entity id', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/5555/children`) .expect(200); expect(body.nextChild).to.eql(null); @@ -244,7 +246,7 @@ export default function ({ getService }: FtrProviderContext) { }); it('returns empty events with an invalid endpoint id', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${entityID}/children?legacyEndpointID=foo`) .expect(200); expect(body.nextChild).to.eql(null); @@ -254,7 +256,7 @@ export default function ({ getService }: FtrProviderContext) { describe('endpoint events', () => { it('returns all children for the origin', async () => { - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=100`) .expect(200); // there are 2 levels in the children part of the tree and 3 nodes for each = @@ -269,7 +271,7 @@ export default function ({ getService }: FtrProviderContext) { // this gets a node should have 3 children which were created in succession so that the timestamps // are ordered correctly to be retrieved in a single call const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; - const { body }: { body: ResolverChildren } = await supertest + const { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) .expect(200); expect(body.childNodes.length).to.eql(3); @@ -281,7 +283,7 @@ export default function ({ getService }: FtrProviderContext) { // this gets a node should have 3 children which were created in succession so that the timestamps // are ordered correctly to be retrieved in a single call const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; - let { body }: { body: ResolverChildren } = await supertest + let { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) .expect(200); expect(body.childNodes.length).to.eql(1); @@ -308,7 +310,7 @@ export default function ({ getService }: FtrProviderContext) { it('gets all children in two queries', async () => { // should get all the children of the origin - let { body }: { body: ResolverChildren } = await supertest + let { body }: { body: SafeResolverChildren } = await supertest .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) .expect(200); expect(body.childNodes.length).to.eql(3); @@ -334,7 +336,7 @@ export default function ({ getService }: FtrProviderContext) { const endpointID = '5a0c957f-b8e7-4538-965e-57e8bb86ad3a'; it('returns ancestors, events, children, and current process lifecycle', async () => { - const { body }: { body: ResolverTree } = await supertest + const { body }: { body: SafeResolverTree } = await supertest .get(`/api/endpoint/resolver/93933?legacyEndpointID=${endpointID}`) .expect(200); expect(body.ancestry.nextAncestor).to.equal(null); @@ -348,7 +350,7 @@ export default function ({ getService }: FtrProviderContext) { describe('endpoint events', () => { it('returns a tree', async () => { - const { body }: { body: ResolverTree } = await supertest + const { body }: { body: SafeResolverTree } = await supertest .get( `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` ) diff --git a/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts b/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts index 7e4d4177affac..c5855281f55c9 100644 --- a/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts +++ b/x-pack/test/security_solution_endpoint_api_int/services/resolver.ts @@ -9,6 +9,7 @@ import { EndpointDocGenerator, Event, } from '../../../plugins/security_solution/common/endpoint/generate_data'; +import { firstNonNullValue } from '../../../plugins/security_solution/common/endpoint/models/ecs_safety_helpers'; import { FtrProviderContext } from '../ftr_provider_context'; export const processEventsIndex = 'logs-endpoint.events.process-default'; @@ -87,7 +88,7 @@ export function ResolverGeneratorProvider({ getService }: FtrProviderContext) { const tree = generator.generateTree(options); const body = tree.allEvents.reduce((array: Array, doc) => { let index = eventsIndex; - if (doc.event.kind === 'alert') { + if (firstNonNullValue(doc.event?.kind) === 'alert') { index = alertsIndex; } /** From 50ec790f71f3c6af14b4d98ac24c4759b86a3041 Mon Sep 17 00:00:00 2001 From: Josh Dover Date: Thu, 10 Sep 2020 12:39:32 -0600 Subject: [PATCH 14/29] Cleanup type output before building new types (#77211) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 95a6de337f62a..7468a49d56959 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "uiFramework:createComponent": "cd packages/kbn-ui-framework && yarn createComponent", "uiFramework:documentComponent": "cd packages/kbn-ui-framework && yarn documentComponent", "kbn:watch": "node scripts/kibana --dev --logging.json=false", - "build:types": "tsc --p tsconfig.types.json", + "build:types": "rm -rf ./target/types && tsc --p tsconfig.types.json", "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept", "kbn:bootstrap": "node scripts/build_ts_refs && node scripts/register_git_hook", "spec_to_console": "node scripts/spec_to_console", From 65abdfffee41e28980c568283845e9fe6fc9bb0b Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Thu, 10 Sep 2020 13:55:40 -0500 Subject: [PATCH 15/29] [Enterprise Search] Update WS Overview logic to use new config data (#77122) * Update AppLogic to use new data structure * Update components to use AppLogic * Remove unused props from logic file * Fix failing tests * Add tests to get 100% converage The test added for app_logic will never happen but hey, 100 is 100. Also added test to recent_activity for 100% coverage * Use non-null assertion operator in logic file TIL this is a thing * Remove test for undefined --- .../workplace_search/app_logic.test.ts | 25 +++++++++++++-- .../workplace_search/app_logic.ts | 32 +++++++++++++++++-- .../views/overview/__mocks__/index.ts | 2 +- .../overview/__mocks__/overview_logic.mock.ts | 18 +++++------ .../views/overview/onboarding_steps.test.tsx | 20 +++++++++--- .../views/overview/onboarding_steps.tsx | 11 ++++--- .../views/overview/organization_stats.tsx | 13 ++++---- .../views/overview/overview.tsx | 12 +++---- .../views/overview/overview_logic.test.ts | 21 ++---------- .../views/overview/overview_logic.ts | 30 ----------------- .../views/overview/recent_activity.test.tsx | 16 ++++++++++ .../views/overview/recent_activity.tsx | 6 ++-- 12 files changed, 116 insertions(+), 90 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.test.ts index bc31b7df5d971..c52eceb2d2fdd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.test.ts @@ -16,7 +16,28 @@ describe('AppLogic', () => { }); const DEFAULT_VALUES = { + account: {}, hasInitialized: false, + isFederatedAuth: true, + organization: {}, + }; + + const expectedLogicValues = { + account: { + canCreateInvitations: true, + canCreatePersonalSources: true, + groups: ['Default', 'Cats'], + id: 'some-id-string', + isAdmin: true, + isCurated: false, + viewedOnboardingPage: true, + }, + hasInitialized: true, + isFederatedAuth: false, + organization: { + defaultOrgName: 'My Organization', + name: 'ACME Donuts', + }, }; it('has expected default values', () => { @@ -27,9 +48,7 @@ describe('AppLogic', () => { it('sets values based on passed props', () => { AppLogic.actions.initializeAppData(DEFAULT_INITIAL_APP_DATA); - expect(AppLogic.values).toEqual({ - hasInitialized: true, - }); + expect(AppLogic.values).toEqual(expectedLogicValues); }); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.ts index 5bf2b41cfc264..f88a00f63f487 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.ts @@ -7,18 +7,26 @@ import { kea, MakeLogicType } from 'kea'; import { IInitialAppData } from '../../../common/types'; -import { IWorkplaceSearchInitialData } from '../../../common/types/workplace_search'; +import { + IOrganization, + IWorkplaceSearchInitialData, + IAccount, +} from '../../../common/types/workplace_search'; export interface IAppValues extends IWorkplaceSearchInitialData { hasInitialized: boolean; + isFederatedAuth: boolean; } export interface IAppActions { - initializeAppData(props: IInitialAppData): void; + initializeAppData(props: IInitialAppData): IInitialAppData; } export const AppLogic = kea>({ actions: { - initializeAppData: ({ workplaceSearch }) => workplaceSearch, + initializeAppData: ({ workplaceSearch, isFederatedAuth }) => ({ + workplaceSearch, + isFederatedAuth, + }), }, reducers: { hasInitialized: [ @@ -27,5 +35,23 @@ export const AppLogic = kea>({ initializeAppData: () => true, }, ], + isFederatedAuth: [ + true, + { + initializeAppData: (_, { isFederatedAuth }) => !!isFederatedAuth, + }, + ], + organization: [ + {} as IOrganization, + { + initializeAppData: (_, { workplaceSearch }) => workplaceSearch!.organization, + }, + ], + account: [ + {} as IAccount, + { + initializeAppData: (_, { workplaceSearch }) => workplaceSearch!.account, + }, + ], }, }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/index.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/index.ts index 9e86993a5289d..9f281a541334e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { setMockValues, mockValues, mockActions } from './overview_logic.mock'; +export { setMockValues, mockOverviewValues, mockActions } from './overview_logic.mock'; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/overview_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/overview_logic.mock.ts index 9ce3021917a21..569e6543ee869 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/overview_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/__mocks__/overview_logic.mock.ts @@ -5,19 +5,18 @@ */ import { IOverviewValues } from '../overview_logic'; -import { IAccount, IOrganization } from '../../../types'; -export const mockValues = { +import { DEFAULT_INITIAL_APP_DATA } from '../../../../../../common/__mocks__'; + +const { workplaceSearch: mockAppValues } = DEFAULT_INITIAL_APP_DATA; + +export const mockOverviewValues = { accountsCount: 0, activityFeed: [], canCreateContentSources: false, - canCreateInvitations: false, - fpAccount: {} as IAccount, hasOrgSources: false, hasUsers: false, - isFederatedAuth: true, isOldAccount: false, - organization: {} as IOrganization, pendingInvitationsCount: 0, personalSourcesCount: 0, sourcesCount: 0, @@ -28,6 +27,8 @@ export const mockActions = { initializeOverview: jest.fn(() => ({})), }; +const mockValues = { ...mockOverviewValues, ...mockAppValues, isFederatedAuth: true }; + jest.mock('kea', () => ({ ...(jest.requireActual('kea') as object), useActions: jest.fn(() => ({ ...mockActions })), @@ -37,8 +38,5 @@ jest.mock('kea', () => ({ import { useValues } from 'kea'; export const setMockValues = (values: object) => { - (useValues as jest.Mock).mockImplementationOnce(() => ({ - ...mockValues, - ...values, - })); + (useValues as jest.Mock).mockImplementation(() => ({ ...mockValues, ...values })); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx index acbc66259c2a1..0f3eee074caef 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.test.tsx @@ -25,6 +25,7 @@ const account = { canCreatePersonalSources: true, groups: [], isCurated: false, + canCreateInvitations: true, }; describe('OnboardingSteps', () => { @@ -60,9 +61,8 @@ describe('OnboardingSteps', () => { describe('Users & Invitations', () => { it('renders 0 users when not on federated auth', () => { setMockValues({ - canCreateInvitations: true, isFederatedAuth: false, - fpAccount: account, + account, accountsCount: 0, hasUsers: false, }); @@ -78,7 +78,7 @@ describe('OnboardingSteps', () => { it('renders completed users state', () => { setMockValues({ isFederatedAuth: false, - fpAccount: account, + account, accountsCount: 1, hasUsers: true, }); @@ -90,7 +90,13 @@ describe('OnboardingSteps', () => { }); it('disables link when the user cannot create invitations', () => { - setMockValues({ isFederatedAuth: false, canCreateInvitations: false }); + setMockValues({ + isFederatedAuth: false, + account: { + ...account, + canCreateInvitations: false, + }, + }); const wrapper = shallow(); expect(wrapper.find(OnboardingCard).last().prop('actionPath')).toBe(undefined); }); @@ -98,6 +104,12 @@ describe('OnboardingSteps', () => { describe('Org Name', () => { it('renders button to change name', () => { + setMockValues({ + organization: { + name: 'foo', + defaultOrgName: 'foo', + }, + }); const wrapper = shallow(); const button = wrapper diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx index 5598123f1c286..0baadfc912ad5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/onboarding_steps.tsx @@ -28,6 +28,7 @@ import { ORG_SOURCES_PATH, USERS_PATH, ORG_SETTINGS_PATH } from '../../routes'; import { ContentSection } from '../../components/shared/content_section'; +import { AppLogic } from '../../app_logic'; import { OverviewLogic } from './overview_logic'; import { OnboardingCard } from './onboarding_card'; @@ -58,16 +59,18 @@ const ONBOARDING_USERS_CARD_DESCRIPTION = i18n.translate( ); export const OnboardingSteps: React.FC = () => { + const { + isFederatedAuth, + organization: { name, defaultOrgName }, + account: { isCurated, canCreateInvitations }, + } = useValues(AppLogic); + const { hasUsers, hasOrgSources, canCreateContentSources, - canCreateInvitations, accountsCount, sourcesCount, - fpAccount: { isCurated }, - organization: { name, defaultOrgName }, - isFederatedAuth, } = useValues(OverviewLogic); const accountsPath = diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx index 4dc762e29deba..6614ac58b0744 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/organization_stats.tsx @@ -14,18 +14,17 @@ import { i18n } from '@kbn/i18n'; import { ContentSection } from '../../components/shared/content_section'; import { ORG_SOURCES_PATH, USERS_PATH } from '../../routes'; +import { AppLogic } from '../../app_logic'; import { OverviewLogic } from './overview_logic'; import { StatisticCard } from './statistic_card'; export const OrganizationStats: React.FC = () => { - const { - sourcesCount, - pendingInvitationsCount, - accountsCount, - personalSourcesCount, - isFederatedAuth, - } = useValues(OverviewLogic); + const { isFederatedAuth } = useValues(AppLogic); + + const { sourcesCount, pendingInvitationsCount, accountsCount, personalSourcesCount } = useValues( + OverviewLogic + ); return ( { - const { initializeOverview } = useActions(OverviewLogic); - const { - dataLoading, - hasUsers, - hasOrgSources, - isOldAccount, organization: { name: orgName, defaultOrgName }, - } = useValues(OverviewLogic); + } = useValues(AppLogic); + + const { initializeOverview } = useActions(OverviewLogic); + const { dataLoading, hasUsers, hasOrgSources, isOldAccount } = useValues(OverviewLogic); useEffect(() => { initializeOverview(); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.test.ts index 6989635064ca9..1ec770e9defce 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.test.ts @@ -9,7 +9,7 @@ import { resetContext } from 'kea'; jest.mock('../../../shared/http', () => ({ HttpLogic: { values: { http: { get: jest.fn() } } } })); import { HttpLogic } from '../../../shared/http'; -import { mockValues } from './__mocks__'; +import { mockOverviewValues } from './__mocks__'; import { OverviewLogic } from './overview_logic'; describe('OverviewLogic', () => { @@ -20,32 +20,19 @@ describe('OverviewLogic', () => { }); it('has expected default values', () => { - expect(OverviewLogic.values).toEqual(mockValues); + expect(OverviewLogic.values).toEqual(mockOverviewValues); }); describe('setServerData', () => { const feed = [{ foo: 'bar' }] as any; - const account = { - id: '1243', - groups: ['Default'], - isAdmin: true, - isCurated: false, - canCreatePersonalSources: true, - viewedOnboardingPage: false, - }; - const org = { name: 'ACME', defaultOrgName: 'Org' }; const data = { accountsCount: 1, activityFeed: feed, canCreateContentSources: true, - canCreateInvitations: true, - fpAccount: account, hasOrgSources: true, hasUsers: true, - isFederatedAuth: false, isOldAccount: true, - organization: org, pendingInvitationsCount: 1, personalSourcesCount: 1, sourcesCount: 1, @@ -60,10 +47,6 @@ describe('OverviewLogic', () => { }); it('will set server values', () => { - expect(OverviewLogic.values.organization).toEqual(org); - expect(OverviewLogic.values.isFederatedAuth).toEqual(false); - expect(OverviewLogic.values.fpAccount).toEqual(account); - expect(OverviewLogic.values.canCreateInvitations).toEqual(true); expect(OverviewLogic.values.hasUsers).toEqual(true); expect(OverviewLogic.values.hasOrgSources).toEqual(true); expect(OverviewLogic.values.canCreateContentSources).toEqual(true); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.ts index 2c6846b6db7db..787d5295db1cf 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/overview_logic.ts @@ -7,24 +7,18 @@ import { kea, MakeLogicType } from 'kea'; import { HttpLogic } from '../../../shared/http'; -import { IAccount, IOrganization } from '../../types'; - import { IFeedActivity } from './recent_activity'; export interface IOverviewServerData { hasUsers: boolean; hasOrgSources: boolean; canCreateContentSources: boolean; - canCreateInvitations: boolean; isOldAccount: boolean; sourcesCount: number; pendingInvitationsCount: number; accountsCount: number; personalSourcesCount: number; activityFeed: IFeedActivity[]; - organization: IOrganization; - isFederatedAuth: boolean; - fpAccount: IAccount; } export interface IOverviewActions { @@ -42,30 +36,6 @@ export const OverviewLogic = kea null, }, reducers: { - organization: [ - {} as IOrganization, - { - setServerData: (_, { organization }) => organization, - }, - ], - isFederatedAuth: [ - true, - { - setServerData: (_, { isFederatedAuth }) => isFederatedAuth, - }, - ], - fpAccount: [ - {} as IAccount, - { - setServerData: (_, { fpAccount }) => fpAccount, - }, - ], - canCreateInvitations: [ - false, - { - setServerData: (_, { canCreateInvitations }) => canCreateInvitations, - }, - ], hasUsers: [ false, { diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx index 22a82af18527d..31613098f9fcc 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.test.tsx @@ -12,6 +12,7 @@ import React from 'react'; import { shallow } from 'enzyme'; import { EuiEmptyPrompt, EuiLink } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; import { RecentActivity, RecentActivityItem } from './recent_activity'; @@ -61,4 +62,19 @@ describe('RecentActivity', () => { expect(wrapper.find('.activity--error__label')).toHaveLength(1); expect(wrapper.find(EuiLink).prop('color')).toEqual('danger'); }); + + it('renders recent activity message for default org name', () => { + setMockValues({ + organization: { + name: 'foo', + defaultOrgName: 'foo', + }, + }); + const wrapper = shallow(); + const emptyPrompt = wrapper.find(EuiEmptyPrompt).dive(); + + expect(emptyPrompt.find(FormattedMessage).prop('defaultMessage')).toEqual( + 'Your organization has no recent activity' + ); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx index 441f45a947a49..0813999c9a078 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/overview/recent_activity.tsx @@ -17,6 +17,7 @@ import { sendTelemetry } from '../../../shared/telemetry'; import { KibanaContext, IKibanaContext } from '../../../index'; import { SOURCE_DETAILS_PATH, getContentSourcePath } from '../../routes'; +import { AppLogic } from '../../app_logic'; import { OverviewLogic } from './overview_logic'; import './recent_activity.scss'; @@ -32,8 +33,9 @@ export interface IFeedActivity { export const RecentActivity: React.FC = () => { const { organization: { name, defaultOrgName }, - activityFeed, - } = useValues(OverviewLogic); + } = useValues(AppLogic); + + const { activityFeed } = useValues(OverviewLogic); return ( Date: Thu, 10 Sep 2020 21:16:07 +0200 Subject: [PATCH 16/29] [Lens] Filters aggregation (#75635) --- ...in-plugins-data-public.querystringinput.md | 2 +- .../components/local_nav/_local_search.scss | 7 - src/plugins/data/public/public.api.md | 2 +- .../ui/query_string_input/_query_bar.scss | 23 +- .../query_string_input/query_string_input.tsx | 6 +- .../data/public/ui/typeahead/constants.ts | 2 +- .../ui/typeahead/suggestions_component.tsx | 1 + .../editor_frame/config_panel/_index.scss | 1 - .../config_panel/_layer_panel.scss | 8 + ...on_popover.scss => dimension_popover.scss} | 7 + .../config_panel/dimension_popover.tsx | 2 + .../dimension_panel/bucket_nesting_editor.tsx | 55 +-- .../dimension_panel/popover_editor.tsx | 7 +- .../indexpattern_datasource/indexpattern.tsx | 1 + .../operations/definitions/cardinality.tsx | 6 +- .../operations/definitions/count.tsx | 6 +- .../definitions/filters/filter_popover.scss | 3 + .../filters/filter_popover.test.tsx | 81 +++++ .../definitions/filters/filter_popover.tsx | 193 ++++++++++ .../definitions/filters/filters.scss | 6 + .../definitions/filters/filters.test.tsx | 284 +++++++++++++++ .../definitions/filters/filters.tsx | 341 ++++++++++++++++++ .../operations/definitions/filters/index.tsx | 7 + .../operations/definitions/index.ts | 76 ++-- .../public/indexpattern_datasource/types.ts | 2 +- 25 files changed, 1050 insertions(+), 79 deletions(-) rename x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/{_dimension_popover.scss => dimension_popover.scss} (51%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.scss create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.test.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.scss create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/index.tsx diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md index 3dbfd9430e913..cf171d9ee9f37 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.querystringinput.md @@ -7,5 +7,5 @@ Signature: ```typescript -QueryStringInput: React.FC> +QueryStringInput: React.FC> ``` diff --git a/packages/kbn-ui-framework/src/components/local_nav/_local_search.scss b/packages/kbn-ui-framework/src/components/local_nav/_local_search.scss index 130807790e987..740ae664c7f5b 100644 --- a/packages/kbn-ui-framework/src/components/local_nav/_local_search.scss +++ b/packages/kbn-ui-framework/src/components/local_nav/_local_search.scss @@ -26,13 +26,6 @@ border-radius: 0; border-left-width: 0; } - -.kuiLocalSearchAssistedInput { - display: flex; - flex: 1 1 100%; - position: relative; -} - /** * 1. em used for right padding so documentation link and query string * won't overlap if the user increases their default browser font size diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 27d4ea49f9eb1..66952c05a50a6 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1479,7 +1479,7 @@ export interface QueryStateChange extends QueryStateChangePartial { // Warning: (ae-missing-release-tag) "QueryStringInput" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const QueryStringInput: React.FC>; +export const QueryStringInput: React.FC>; // @public (undocumented) export type QuerySuggestion = QuerySuggestionBasic | QuerySuggestionField; diff --git a/src/plugins/data/public/ui/query_string_input/_query_bar.scss b/src/plugins/data/public/ui/query_string_input/_query_bar.scss index 00895ec49003b..1ff24c61954e7 100644 --- a/src/plugins/data/public/ui/query_string_input/_query_bar.scss +++ b/src/plugins/data/public/ui/query_string_input/_query_bar.scss @@ -8,30 +8,37 @@ border-right: none !important; } +.kbnQueryBar__textareaWrap { + overflow: visible !important; // Override EUI form control + display: flex; + flex: 1 1 100%; + position: relative; +} + .kbnQueryBar__textarea { z-index: $euiZContentMenu; resize: none !important; // When in the group, it will autosize - height: $euiSizeXXL; + height: $euiFormControlHeight; // Unlike most inputs within layout control groups, the text area still needs a border. // These adjusts help it sit above the control groups shadow to line up correctly. - padding-top: $euiSizeS + 3px !important; - transform: translateY(-2px); - padding: $euiSizeS - 1px; + padding: $euiSizeS; + padding-top: $euiSizeS + 3px; + transform: translateY(-1px) translateX(-1px); - &:not(:focus) { + &:not(:focus):not(:invalid) { @include euiYScrollWithShadows; + } + + &:not(:focus) { white-space: nowrap; overflow-y: hidden; overflow-x: hidden; - border: none; - box-shadow: none; } // When focused, let it scroll &:focus { overflow-x: auto; overflow-y: auto; - width: calc(100% + 1px); // To overtake the group's fake border white-space: normal; } } diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx index 0bfac2a07a7eb..f159cac664a9e 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx @@ -19,6 +19,7 @@ import React, { Component, RefObject, createRef } from 'react'; import { i18n } from '@kbn/i18n'; + import classNames from 'classnames'; import { EuiTextArea, @@ -63,6 +64,7 @@ interface Props { dataTestSubj?: string; size?: SuggestionsListSize; className?: string; + isInvalid?: boolean; } interface State { @@ -591,6 +593,7 @@ export class QueryStringInputUI extends Component { 'euiFormControlLayout euiFormControlLayout--group kbnQueryBar__wrap', this.props.className ); + return (
{this.props.prepend} @@ -607,7 +610,7 @@ export class QueryStringInputUI extends Component { >
{ } role="textbox" data-test-subj={this.props.dataTestSubj || 'queryInput'} + isInvalid={this.props.isInvalid} > {this.getQueryString()} diff --git a/src/plugins/data/public/ui/typeahead/constants.ts b/src/plugins/data/public/ui/typeahead/constants.ts index 08f9bd23e16f3..0e28891a14535 100644 --- a/src/plugins/data/public/ui/typeahead/constants.ts +++ b/src/plugins/data/public/ui/typeahead/constants.ts @@ -33,4 +33,4 @@ export const SUGGESTIONS_LIST_REQUIRED_BOTTOM_SPACE = 250; * A distance in px to display suggestions list right under the query input without a gap * @public */ -export const SUGGESTIONS_LIST_REQUIRED_TOP_OFFSET = 2; +export const SUGGESTIONS_LIST_REQUIRED_TOP_OFFSET = 1; diff --git a/src/plugins/data/public/ui/typeahead/suggestions_component.tsx b/src/plugins/data/public/ui/typeahead/suggestions_component.tsx index dc7c55374f1d5..50ed9e9542d36 100644 --- a/src/plugins/data/public/ui/typeahead/suggestions_component.tsx +++ b/src/plugins/data/public/ui/typeahead/suggestions_component.tsx @@ -154,6 +154,7 @@ export class SuggestionsComponent extends Component { const StyledSuggestionsListDiv = styled.div` ${(props: { queryBarRect: DOMRect; verticalListPosition: string }) => ` position: absolute; + z-index: 4001; left: ${props.queryBarRect.left}px; width: ${props.queryBarRect.width}px; ${props.verticalListPosition}`} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_index.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_index.scss index 5b968abd0c061..954fbfadf159b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_index.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_index.scss @@ -1,3 +1,2 @@ @import 'config_panel'; -@import 'dimension_popover'; @import 'layer_panel'; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_layer_panel.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_layer_panel.scss index 62bc6d7ed7cc8..ab53ff983ca26 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_layer_panel.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_layer_panel.scss @@ -43,6 +43,14 @@ min-height: $euiSizeXXL; } +.lnsLayerPanel__anchor { + width: 100%; +} + +.lnsLayerPanel__dndGrab { + padding: $euiSizeS; +} + .lnsLayerPanel__styleEditor { width: $euiSize * 30; padding: $euiSizeS; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_dimension_popover.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.scss similarity index 51% rename from x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_dimension_popover.scss rename to x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.scss index 691cda9ff0d79..98036c7f31bd9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/_dimension_popover.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.scss @@ -9,3 +9,10 @@ display: block; word-break: break-word; } + +// todo: remove after closing https://github.com/elastic/eui/issues/3548 +.lnsDimensionPopover__fixTranslateDnd { + // sass-lint:disable-block no-important + transform: none !important; +} + diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.tsx index 8d31e1bcc2e6a..a90bd8122d18e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_popover.tsx @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import './dimension_popover.scss'; import React from 'react'; import { EuiPopover } from '@elastic/eui'; @@ -31,6 +32,7 @@ export function DimensionPopover({ = { + terms: i18n.translate('xpack.lens.indexPattern.groupingOverallTerms', { + defaultMessage: 'Overall top {field}', + values: { field: fieldName }, + }), + filters: i18n.translate('xpack.lens.indexPattern.groupingOverallFilters', { + defaultMessage: 'Top values for each custom query', + }), + date_histogram: i18n.translate('xpack.lens.indexPattern.groupingOverallDateHistogram', { + defaultMessage: 'Top values for each {field}', + values: { field: fieldName }, + }), + }; + + const bottomLevelCopy: Record = { + terms: i18n.translate('xpack.lens.indexPattern.groupingSecondTerms', { + defaultMessage: 'Top values for each {target}', + values: { target: target.fieldName }, + }), + filters: i18n.translate('xpack.lens.indexPattern.groupingSecondFilters', { + defaultMessage: 'Overall top {target}', + values: { target: target.fieldName }, + }), + date_histogram: i18n.translate('xpack.lens.indexPattern.groupingSecondDateHistogram', { + defaultMessage: 'Overall top {target}', + values: { target: target.fieldName }, + }), + }; + return ( <> @@ -73,34 +104,14 @@ export function BucketNestingEditor({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx index 038b51b922286..d5f0110f071f1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx @@ -160,6 +160,11 @@ export function PopoverEditor(props: PopoverEditorProps) { compatibleWithCurrentField ? '' : ' incompatible' }`, onClick() { + // todo: when moving from terms agg to filters, we want to create a filter `$field.name : *` + // it probably has to be re-thought when removing the field name. + const isTermsToFilters = + selectedColumn?.operationType === 'terms' && operationType === 'filters'; + if (!selectedColumn || !compatibleWithCurrentField) { const possibleFields = fieldByOperation[operationType] || []; @@ -186,7 +191,7 @@ export function PopoverEditor(props: PopoverEditorProps) { trackUiEvent(`indexpattern_dimension_operation_${operationType}`); return; } - if (incompatibleSelectedOperationType) { + if (incompatibleSelectedOperationType && !isTermsToFilters) { setInvalidOperationType(null); } if (selectedColumn.operationType === operationType) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index e2ca933504849..3b3750cf7c560 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -263,6 +263,7 @@ export function getIndexPatternDatasource({ data, savedObjects: core.savedObjects, docLinks: core.docLinks, + http: core.http, }} > ({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 4e081da2c6dc9..bb1aef856de78 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -49,7 +49,11 @@ export const countOperation: OperationDefinition = { scale: 'ratio', sourceField: field.name, params: - previousColumn && previousColumn.dataType === 'number' ? previousColumn.params : undefined, + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params + ? previousColumn.params + : undefined, }; }, toEsAggsConfig: (column, columnId) => ({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.scss new file mode 100644 index 0000000000000..6838812e4b999 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.scss @@ -0,0 +1,3 @@ +.lnsIndexPatternDimensionEditor__filtersEditor { + width: $euiSize * 60; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.test.tsx new file mode 100644 index 0000000000000..4d4b4018d75a7 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.test.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { MouseEventHandler } from 'react'; +import { shallow } from 'enzyme'; +import { act } from 'react-dom/test-utils'; +import { EuiPopover, EuiLink } from '@elastic/eui'; +import { createMockedIndexPattern } from '../../../mocks'; +import { FilterPopover, QueryInput, LabelInput } from './filter_popover'; + +jest.mock('.', () => ({ + isQueryValid: () => true, + defaultLabel: 'label', +})); + +const defaultProps = { + filter: { + input: { query: 'bytes >= 1', language: 'kuery' }, + label: 'More than one', + id: '1', + }, + setFilter: jest.fn(), + indexPattern: createMockedIndexPattern(), + Button: ({ onClick }: { onClick: MouseEventHandler }) => ( + trigger + ), + isOpenByCreation: true, + setIsOpenByCreation: jest.fn(), +}; + +describe('filter popover', () => { + jest.mock('../../../../../../../../src/plugins/data/public', () => ({ + QueryStringInput: () => { + return 'QueryStringInput'; + }, + })); + it('should be open if is open by creation', () => { + const setIsOpenByCreation = jest.fn(); + const instance = shallow( + + ); + expect(instance.find(EuiPopover).prop('isOpen')).toEqual(true); + act(() => { + instance.find(EuiPopover).prop('closePopover')!(); + }); + instance.update(); + expect(setIsOpenByCreation).toHaveBeenCalledWith(false); + }); + it('should call setFilter when modifying QueryInput', () => { + const setFilter = jest.fn(); + const instance = shallow(); + instance.find(QueryInput).prop('onChange')!({ + query: 'modified : query', + language: 'lucene', + }); + expect(setFilter).toHaveBeenCalledWith({ + input: { + language: 'lucene', + query: 'modified : query', + }, + label: 'More than one', + id: '1', + }); + }); + it('should call setFilter when modifying LabelInput', () => { + const setFilter = jest.fn(); + const instance = shallow(); + instance.find(LabelInput).prop('onChange')!('Modified label'); + expect(setFilter).toHaveBeenCalledWith({ + input: { + language: 'kuery', + query: 'bytes >= 1', + }, + label: 'Modified label', + id: '1', + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx new file mode 100644 index 0000000000000..cdfa19f53a13a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filter_popover.tsx @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import './filter_popover.scss'; + +import React, { MouseEventHandler, useState } from 'react'; +import { useDebounce } from 'react-use'; +import { EuiPopover, EuiFieldText, EuiSpacer, keys } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FilterValue, defaultLabel, isQueryValid } from '.'; +import { IndexPattern } from '../../../types'; +import { QueryStringInput, Query } from '../../../../../../../../src/plugins/data/public'; + +export const FilterPopover = ({ + filter, + setFilter, + indexPattern, + Button, + isOpenByCreation, + setIsOpenByCreation, +}: { + filter: FilterValue; + setFilter: Function; + indexPattern: IndexPattern; + Button: React.FunctionComponent<{ onClick: MouseEventHandler }>; + isOpenByCreation: boolean; + setIsOpenByCreation: Function; +}) => { + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const inputRef = React.useRef(); + + const setPopoverOpen = (isOpen: boolean) => { + setIsPopoverOpen(isOpen); + setIsOpenByCreation(isOpen); + }; + + const setFilterLabel = (label: string) => setFilter({ ...filter, label }); + const setFilterQuery = (input: Query) => setFilter({ ...filter, input }); + + const getPlaceholder = (query: Query['query']) => { + if (query === '') { + return defaultLabel; + } + if (query === 'object') return JSON.stringify(query); + else { + return String(query); + } + }; + + return ( + { + setPopoverOpen(false); + }} + button={ +