From 875c47593618b4b55622ca90beb387a0cbc837dc Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Date: Thu, 15 Apr 2021 17:58:51 -0400 Subject: [PATCH 01/25] happy it is fix (#97313) --- .../timelines/containers/index.test.tsx | 18 ++++++-- .../public/timelines/containers/index.tsx | 46 ++++++++++--------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx index b24a50a516325..ddcecb885a8ff 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx @@ -230,13 +230,25 @@ describe('useTimelineEvents', () => { // useEffect on params request await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); + rerender({ + ...props, + startDate, + endDate, + language: 'eql', + eqlOptions: { + eventCategoryField: 'category', + tiebreakerField: '', + timestampField: '@timestamp', + query: 'find it EQL', + size: 100, + }, + }); // useEffect on params request await waitForNextUpdate(); - expect(mockSearch).toHaveBeenCalledTimes(2); + mockSearch.mockReset(); result.current[1].loadPage(4); await waitForNextUpdate(); - expect(mockSearch).toHaveBeenCalledTimes(3); + expect(mockSearch).toHaveBeenCalledTimes(1); }); }); }); diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx index ab4b4358fd326..5f464b5ed943f 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx @@ -98,6 +98,29 @@ export const initSortDefault = [ }, ]; +const deStructureEqlOptions = (eqlOptions?: EqlOptionsSelected) => ({ + ...(!isEmpty(eqlOptions?.eventCategoryField) + ? { + eventCategoryField: eqlOptions?.eventCategoryField, + } + : {}), + ...(!isEmpty(eqlOptions?.size) + ? { + size: eqlOptions?.size, + } + : {}), + ...(!isEmpty(eqlOptions?.tiebreakerField) + ? { + tiebreakerField: eqlOptions?.tiebreakerField, + } + : {}), + ...(!isEmpty(eqlOptions?.timestampField) + ? { + timestampField: eqlOptions?.timestampField, + } + : {}), +}); + export const useTimelineEvents = ({ docValueFields, endDate, @@ -293,26 +316,7 @@ export const useTimelineEvents = ({ querySize: prevRequest?.pagination.querySize ?? 0, sort: prevRequest?.sort ?? initSortDefault, timerange: prevRequest?.timerange ?? {}, - ...(!isEmpty(prevEqlRequest?.eventCategoryField) - ? { - eventCategoryField: prevEqlRequest?.eventCategoryField, - } - : {}), - ...(!isEmpty(prevEqlRequest?.size) - ? { - size: prevEqlRequest?.size, - } - : {}), - ...(!isEmpty(prevEqlRequest?.tiebreakerField) - ? { - tiebreakerField: prevEqlRequest?.tiebreakerField, - } - : {}), - ...(!isEmpty(prevEqlRequest?.timestampField) - ? { - timestampField: prevEqlRequest?.timestampField, - } - : {}), + ...deStructureEqlOptions(prevEqlRequest), }; const currentSearchParameters = { @@ -325,7 +329,7 @@ export const useTimelineEvents = ({ from: startDate, to: endDate, }, - ...(eqlOptions ? eqlOptions : {}), + ...deStructureEqlOptions(eqlOptions), }; const newActivePage = deepEqual(prevSearchParameters, currentSearchParameters) From 689c4d40cda3e4beb86fa87e790ecbba73a44109 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 15 Apr 2021 17:12:52 -0700 Subject: [PATCH 02/25] [ci/baseline] check public API doc count to populate baseline metrics (#97320) Co-authored-by: spalger --- .ci/Jenkinsfile_baseline_capture | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile_baseline_capture b/.ci/Jenkinsfile_baseline_capture index 7fefbbb26fd12..b729f5d9da082 100644 --- a/.ci/Jenkinsfile_baseline_capture +++ b/.ci/Jenkinsfile_baseline_capture @@ -36,6 +36,7 @@ kibanaPipeline(timeoutMinutes: 210) { tasks([ kibanaPipeline.functionalTestProcess('oss-baseline', './test/scripts/jenkins_baseline.sh'), kibanaPipeline.functionalTestProcess('xpack-baseline', './test/scripts/jenkins_xpack_baseline.sh'), + kibanaPipeline.scriptTask('Check Public API Docs', 'test/scripts/checks/plugin_public_api_docs.sh'), ]) } } From 9d0b495a30c54b1b5757e6bdb096b78112f843f1 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 15 Apr 2021 17:13:38 -0700 Subject: [PATCH 03/25] [ci] ship Jest unit test junit with runbld in jest worker (#97197) Co-authored-by: Brian Seeders Co-authored-by: spalger --- vars/kibanaPipeline.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 466a04d9b6b39..b8afdb9cde3ef 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -468,7 +468,13 @@ def allCiTasks() { }, jest: { workers.ci(name: 'jest', size: 'n2-standard-16', ramDisk: false) { - scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh')() + catchErrors { + scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh')() + } + + catchErrors { + runbld.junit() + } } }, ]) From 6e5c9278baa586e03323a871c45e5ca74d328292 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Thu, 15 Apr 2021 21:01:28 -0400 Subject: [PATCH 04/25] [Fleet] Use 'hosted agent policy' naming in UI & API messages (#97287) ## Summary ### What this PR _does_ change - [x] Replace all UI & API instances of "managed policy" with "hosted agent policy" #93504 - [x] Replace all UI & API instances of "unmanaged policy" with "regular agent policy" #93504 - [x] Update most variable names containing `managed` to `hosted`
screenshots Screen Shot 2021-04-15 at 10 54 01 AM Screen Shot 2021-04-15 at 11 40 47 AM Screen Shot 2021-04-15 at 11 40 59 AM
### What this PR _does not_ change - [ ] The `is_managed` property of the agent policy saved object* - [ ] The updated error messages in https://github.com/elastic/kibana/issues/92591
*why not change the is_managed property?

Changing the property will require a migration from 7.12. That's not a problem, but we haven't decided on a new name/approach yet. Rather than update to is_hosted: boolean now and potentially something like managed_by: enum later, we'll update it when we have a decision.

TL;DR: It's a trivial update to make later and a minuscule amount of debt to pay until then. Less than cost of doing it twice.

### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../agent_policy/details_page/index.tsx | 4 +- .../sections/agents/agent_list_page/index.tsx | 4 +- .../fleet/server/services/agent_policy.ts | 6 +- .../fleet/server/services/agents/crud.ts | 5 +- .../server/services/agents/reassign.test.ts | 85 +++++------ .../fleet/server/services/agents/reassign.ts | 4 +- .../server/services/agents/unenroll.test.ts | 140 ++++++++---------- .../fleet/server/services/agents/unenroll.ts | 2 +- .../fleet/server/services/agents/upgrade.ts | 16 +- .../fleet/server/services/package_policy.ts | 2 +- .../translations/translations/ja-JP.json | 2 +- .../translations/translations/zh-CN.json | 2 +- .../apis/agent_policy/agent_policy.ts | 26 ++-- .../apis/agents/reassign.ts | 16 +- .../apis/agents/unenroll.ts | 20 +-- .../apis/agents/upgrade.ts | 26 ++-- .../apis/package_policy/create.ts | 20 +-- .../apis/package_policy/delete.ts | 12 +- .../apis/package_policy/update.ts | 2 +- 19 files changed, 191 insertions(+), 203 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/index.tsx index 3e6ca5944c380..65cf62a279a22 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/index.tsx @@ -106,9 +106,9 @@ export const AgentPolicyDetailsPage: React.FunctionComponent = () => { {agentPolicy?.is_managed && ( = () => { if (!agent.policy_id) return true; const agentPolicy = agentPoliciesIndexedById[agent.policy_id]; - const isManaged = agentPolicy?.is_managed === true; - return !isManaged; + const isHosted = agentPolicy?.is_managed === true; + return !isHosted; }; const columns = [ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index e398eb4527f52..6237951805547 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -476,7 +476,7 @@ class AgentPolicyService { } if (oldAgentPolicy.is_managed && !options?.force) { - throw new IngestManagerError(`Cannot update integrations of managed policy ${id}`); + throw new IngestManagerError(`Cannot update integrations of hosted agent policy ${id}`); } return await this._update( @@ -507,7 +507,7 @@ class AgentPolicyService { } if (oldAgentPolicy.is_managed && !options?.force) { - throw new IngestManagerError(`Cannot remove integrations of managed policy ${id}`); + throw new IngestManagerError(`Cannot remove integrations of hosted agent policy ${id}`); } return await this._update( @@ -550,7 +550,7 @@ class AgentPolicyService { } if (agentPolicy.is_managed) { - throw new AgentPolicyDeletionError(`Cannot delete managed policy ${id}`); + throw new AgentPolicyDeletionError(`Cannot delete hosted agent policy ${id}`); } const { diff --git a/x-pack/plugins/fleet/server/services/agents/crud.ts b/x-pack/plugins/fleet/server/services/agents/crud.ts index a23efa1e50fc0..b8ce7c36e507f 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud.ts @@ -255,9 +255,10 @@ export async function getAgentByAccessAPIKeyId( q: `access_api_key_id:${escapeSearchQueryPhrase(accessAPIKeyId)}`, }); - const agent = searchHitToAgent(res.body.hits.hits[0]); + const searchHit = res.body.hits.hits[0]; + const agent = searchHit && searchHitToAgent(searchHit); - if (!agent) { + if (!searchHit || !agent) { throw new AgentNotFoundError('Agent not found'); } if (agent.access_api_key_id !== accessAPIKeyId) { diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts index f040ba57c38be..4dfc29df8c398 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts @@ -13,63 +13,63 @@ import { AgentReassignmentError } from '../../errors'; import { reassignAgent, reassignAgents } from './reassign'; -const agentInManagedDoc = { - _id: 'agent-in-managed-policy', - _source: { policy_id: 'managed-agent-policy' }, +const agentInHostedDoc = { + _id: 'agent-in-hosted-policy', + _source: { policy_id: 'hosted-agent-policy' }, }; -const agentInManagedDoc2 = { - _id: 'agent-in-managed-policy2', - _source: { policy_id: 'managed-agent-policy' }, +const agentInHostedDoc2 = { + _id: 'agent-in-hosted-policy2', + _source: { policy_id: 'hosted-agent-policy' }, }; -const agentInUnmanagedDoc = { - _id: 'agent-in-unmanaged-policy', - _source: { policy_id: 'unmanaged-agent-policy' }, +const agentInRegularDoc = { + _id: 'agent-in-regular-policy', + _source: { policy_id: 'regular-agent-policy' }, }; -const unmanagedAgentPolicySO = { - id: 'unmanaged-agent-policy', +const regularAgentPolicySO = { + id: 'regular-agent-policy', attributes: { is_managed: false }, } as SavedObject; -const unmanagedAgentPolicySO2 = { - id: 'unmanaged-agent-policy-2', +const regularAgentPolicySO2 = { + id: 'regular-agent-policy-2', attributes: { is_managed: false }, } as SavedObject; -const managedAgentPolicySO = { - id: 'managed-agent-policy', +const hostedAgentPolicySO = { + id: 'hosted-agent-policy', attributes: { is_managed: true }, } as SavedObject; describe('reassignAgent (singular)', () => { - it('can reassign from unmanaged policy to unmanaged', async () => { + it('can reassign from regular agent policy to regular', async () => { const { soClient, esClient } = createClientsMock(); - await reassignAgent(soClient, esClient, agentInUnmanagedDoc._id, unmanagedAgentPolicySO.id); + await reassignAgent(soClient, esClient, agentInRegularDoc._id, regularAgentPolicySO.id); // calls ES update with correct values expect(esClient.update).toBeCalledTimes(1); const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInUnmanagedDoc._id); - expect(calledWith[0]?.body?.doc).toHaveProperty('policy_id', unmanagedAgentPolicySO.id); + expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); + expect(calledWith[0]?.body?.doc).toHaveProperty('policy_id', regularAgentPolicySO.id); }); - it('cannot reassign from unmanaged policy to managed', async () => { + it('cannot reassign from regular agent policy to hosted', async () => { const { soClient, esClient } = createClientsMock(); await expect( - reassignAgent(soClient, esClient, agentInUnmanagedDoc._id, managedAgentPolicySO.id) + reassignAgent(soClient, esClient, agentInRegularDoc._id, hostedAgentPolicySO.id) ).rejects.toThrowError(AgentReassignmentError); // does not call ES update expect(esClient.update).toBeCalledTimes(0); }); - it('cannot reassign from managed policy', async () => { + it('cannot reassign from hosted agent policy', async () => { const { soClient, esClient } = createClientsMock(); await expect( - reassignAgent(soClient, esClient, agentInManagedDoc._id, unmanagedAgentPolicySO.id) + reassignAgent(soClient, esClient, agentInHostedDoc._id, regularAgentPolicySO.id) ).rejects.toThrowError(AgentReassignmentError); // does not call ES update expect(esClient.update).toBeCalledTimes(0); await expect( - reassignAgent(soClient, esClient, agentInManagedDoc._id, managedAgentPolicySO.id) + reassignAgent(soClient, esClient, agentInHostedDoc._id, hostedAgentPolicySO.id) ).rejects.toThrowError(AgentReassignmentError); // does not call ES update expect(esClient.update).toBeCalledTimes(0); @@ -77,22 +77,17 @@ describe('reassignAgent (singular)', () => { }); describe('reassignAgents (plural)', () => { - it('agents in managed policies are not updated', async () => { + it('agents in hosted policies are not updated', async () => { const { soClient, esClient } = createClientsMock(); - const idsToReassign = [agentInUnmanagedDoc._id, agentInManagedDoc._id, agentInManagedDoc2._id]; - await reassignAgents( - soClient, - esClient, - { agentIds: idsToReassign }, - unmanagedAgentPolicySO2.id - ); + const idsToReassign = [agentInRegularDoc._id, agentInHostedDoc._id, agentInHostedDoc2._id]; + await reassignAgents(soClient, esClient, { agentIds: idsToReassign }, regularAgentPolicySO2.id); // calls ES update with correct values const calledWith = esClient.bulk.mock.calls[0][0]; - // only 1 are unmanaged and bulk write two line per update + // only 1 are regular and bulk write two line per update expect(calledWith.body.length).toBe(2); // @ts-expect-error - expect(calledWith.body[0].update._id).toEqual(agentInUnmanagedDoc._id); + expect(calledWith.body[0].update._id).toEqual(agentInRegularDoc._id); }); }); @@ -112,12 +107,12 @@ function createClientsMock() { }); soClientMock.get.mockImplementation(async (_, id) => { switch (id) { - case unmanagedAgentPolicySO.id: - return unmanagedAgentPolicySO; - case managedAgentPolicySO.id: - return managedAgentPolicySO; - case unmanagedAgentPolicySO2.id: - return unmanagedAgentPolicySO2; + case regularAgentPolicySO.id: + return regularAgentPolicySO; + case hostedAgentPolicySO.id: + return hostedAgentPolicySO; + case regularAgentPolicySO2.id: + return regularAgentPolicySO2; default: throw new Error(`${id} not found`); } @@ -133,17 +128,17 @@ function createClientsMock() { esClientMock.mget.mockImplementation(async () => { return { body: { - docs: [agentInManagedDoc, agentInUnmanagedDoc, agentInManagedDoc2], + docs: [agentInHostedDoc, agentInRegularDoc, agentInHostedDoc2], }, }; }); // @ts-expect-error esClientMock.get.mockImplementation(async ({ id }) => { switch (id) { - case agentInManagedDoc._id: - return { body: agentInManagedDoc }; - case agentInUnmanagedDoc._id: - return { body: agentInUnmanagedDoc }; + case agentInHostedDoc._id: + return { body: agentInHostedDoc }; + case agentInRegularDoc._id: + return { body: agentInRegularDoc }; default: throw new Error(`${id} not found`); } diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.ts b/x-pack/plugins/fleet/server/services/agents/reassign.ts index 2d94e8b9e9247..4c95d19e2f13a 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.ts @@ -57,14 +57,14 @@ export async function reassignAgentIsAllowed( const agentPolicy = await getAgentPolicyForAgent(soClient, esClient, agentId); if (agentPolicy?.is_managed) { throw new AgentReassignmentError( - `Cannot reassign an agent from managed agent policy ${agentPolicy.id}` + `Cannot reassign an agent from hosted agent policy ${agentPolicy.id}` ); } const newAgentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId); if (newAgentPolicy?.is_managed) { throw new AgentReassignmentError( - `Cannot reassign an agent to managed agent policy ${newAgentPolicy.id}` + `Cannot reassign an agent to hosted agent policy ${newAgentPolicy.id}` ); } diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts index 938ece1364b40..24a3dea3bcb91 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts @@ -13,82 +13,82 @@ import { AgentUnenrollmentError } from '../../errors'; import { unenrollAgent, unenrollAgents } from './unenroll'; -const agentInManagedDoc = { - _id: 'agent-in-managed-policy', - _source: { policy_id: 'managed-agent-policy' }, +const agentInHostedDoc = { + _id: 'agent-in-hosted-policy', + _source: { policy_id: 'hosted-agent-policy' }, }; -const agentInUnmanagedDoc = { - _id: 'agent-in-unmanaged-policy', - _source: { policy_id: 'unmanaged-agent-policy' }, +const agentInRegularDoc = { + _id: 'agent-in-regular-policy', + _source: { policy_id: 'regular-agent-policy' }, }; -const agentInUnmanagedDoc2 = { - _id: 'agent-in-unmanaged-policy2', - _source: { policy_id: 'unmanaged-agent-policy' }, +const agentInRegularDoc2 = { + _id: 'agent-in-regular-policy2', + _source: { policy_id: 'regular-agent-policy' }, }; -const unmanagedAgentPolicySO = { - id: 'unmanaged-agent-policy', +const regularAgentPolicySO = { + id: 'regular-agent-policy', attributes: { is_managed: false }, } as SavedObject; -const managedAgentPolicySO = { - id: 'managed-agent-policy', +const hostedAgentPolicySO = { + id: 'hosted-agent-policy', attributes: { is_managed: true }, } as SavedObject; describe('unenrollAgent (singular)', () => { - it('can unenroll from unmanaged policy', async () => { + it('can unenroll from regular agent policy', async () => { const { soClient, esClient } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInUnmanagedDoc._id); + await unenrollAgent(soClient, esClient, agentInRegularDoc._id); // calls ES update with correct values expect(esClient.update).toBeCalledTimes(1); const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInUnmanagedDoc._id); + expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); expect(calledWith[0]?.body).toHaveProperty('doc.unenrollment_started_at'); }); - it('cannot unenroll from managed policy by default', async () => { + it('cannot unenroll from hosted agent policy by default', async () => { const { soClient, esClient } = createClientMock(); - await expect(unenrollAgent(soClient, esClient, agentInManagedDoc._id)).rejects.toThrowError( + await expect(unenrollAgent(soClient, esClient, agentInHostedDoc._id)).rejects.toThrowError( AgentUnenrollmentError ); // does not call ES update expect(esClient.update).toBeCalledTimes(0); }); - it('cannot unenroll from managed policy with revoke=true', async () => { + it('cannot unenroll from hosted agent policy with revoke=true', async () => { const { soClient, esClient } = createClientMock(); await expect( - unenrollAgent(soClient, esClient, agentInManagedDoc._id, { revoke: true }) + unenrollAgent(soClient, esClient, agentInHostedDoc._id, { revoke: true }) ).rejects.toThrowError(AgentUnenrollmentError); // does not call ES update expect(esClient.update).toBeCalledTimes(0); }); - it('can unenroll from managed policy with force=true', async () => { + it('can unenroll from hosted agent policy with force=true', async () => { const { soClient, esClient } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInManagedDoc._id, { force: true }); + await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true }); // calls ES update with correct values expect(esClient.update).toBeCalledTimes(1); const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInManagedDoc._id); + expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); expect(calledWith[0]?.body).toHaveProperty('doc.unenrollment_started_at'); }); - it('can unenroll from managed policy with force=true and revoke=true', async () => { + it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { const { soClient, esClient } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInManagedDoc._id, { force: true, revoke: true }); + await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true, revoke: true }); // calls ES update with correct values expect(esClient.update).toBeCalledTimes(1); const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInManagedDoc._id); + expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); expect(calledWith[0]?.body).toHaveProperty('doc.unenrolled_at'); }); }); describe('unenrollAgents (plural)', () => { - it('can unenroll from an unmanaged policy', async () => { + it('can unenroll from an regular agent policy', async () => { const { soClient, esClient } = createClientMock(); - const idsToUnenroll = [agentInUnmanagedDoc._id, agentInUnmanagedDoc2._id]; + const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); // calls ES update with correct values @@ -102,37 +102,29 @@ describe('unenrollAgents (plural)', () => { expect(doc).toHaveProperty('unenrollment_started_at'); } }); - it('cannot unenroll from a managed policy by default', async () => { + it('cannot unenroll from a hosted agent policy by default', async () => { const { soClient, esClient } = createClientMock(); - const idsToUnenroll = [ - agentInUnmanagedDoc._id, - agentInManagedDoc._id, - agentInUnmanagedDoc2._id, - ]; + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); // calls ES update with correct values - const onlyUnmanaged = [agentInUnmanagedDoc._id, agentInUnmanagedDoc2._id]; + const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; const calledWith = esClient.bulk.mock.calls[1][0]; const ids = calledWith?.body .filter((i: any) => i.update !== undefined) .map((i: any) => i.update._id); const docs = calledWith?.body.filter((i: any) => i.doc).map((i: any) => i.doc); - expect(ids).toEqual(onlyUnmanaged); + expect(ids).toEqual(onlyRegular); for (const doc of docs) { expect(doc).toHaveProperty('unenrollment_started_at'); } }); - it('cannot unenroll from a managed policy with revoke=true', async () => { + it('cannot unenroll from a hosted agent policy with revoke=true', async () => { const { soClient, esClient } = createClientMock(); - const idsToUnenroll = [ - agentInUnmanagedDoc._id, - agentInManagedDoc._id, - agentInUnmanagedDoc2._id, - ]; + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; const unenrolledResponse = await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll, @@ -141,39 +133,35 @@ describe('unenrollAgents (plural)', () => { expect(unenrolledResponse.items).toMatchObject([ { - id: 'agent-in-unmanaged-policy', + id: 'agent-in-regular-policy', success: true, }, { - id: 'agent-in-managed-policy', + id: 'agent-in-hosted-policy', success: false, }, { - id: 'agent-in-unmanaged-policy2', + id: 'agent-in-regular-policy2', success: true, }, ]); // calls ES update with correct values - const onlyUnmanaged = [agentInUnmanagedDoc._id, agentInUnmanagedDoc2._id]; + const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; const calledWith = esClient.bulk.mock.calls[0][0]; const ids = calledWith?.body .filter((i: any) => i.update !== undefined) .map((i: any) => i.update._id); const docs = calledWith?.body.filter((i: any) => i.doc).map((i: any) => i.doc); - expect(ids).toEqual(onlyUnmanaged); + expect(ids).toEqual(onlyRegular); for (const doc of docs) { expect(doc).toHaveProperty('unenrolled_at'); } }); - it('can unenroll from managed policy with force=true', async () => { + it('can unenroll from hosted agent policy with force=true', async () => { const { soClient, esClient } = createClientMock(); - const idsToUnenroll = [ - agentInUnmanagedDoc._id, - agentInManagedDoc._id, - agentInUnmanagedDoc2._id, - ]; + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll, force: true }); // calls ES update with correct values @@ -188,14 +176,10 @@ describe('unenrollAgents (plural)', () => { } }); - it('can unenroll from managed policy with force=true and revoke=true', async () => { + it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { const { soClient, esClient } = createClientMock(); - const idsToUnenroll = [ - agentInUnmanagedDoc._id, - agentInManagedDoc._id, - agentInUnmanagedDoc2._id, - ]; + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; const unenrolledResponse = await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll, @@ -205,15 +189,15 @@ describe('unenrollAgents (plural)', () => { expect(unenrolledResponse.items).toMatchObject([ { - id: 'agent-in-unmanaged-policy', + id: 'agent-in-regular-policy', success: true, }, { - id: 'agent-in-managed-policy', + id: 'agent-in-hosted-policy', success: true, }, { - id: 'agent-in-unmanaged-policy2', + id: 'agent-in-regular-policy2', success: true, }, ]); @@ -248,10 +232,10 @@ function createClientMock() { soClientMock.get.mockImplementation(async (_, id) => { switch (id) { - case unmanagedAgentPolicySO.id: - return unmanagedAgentPolicySO; - case managedAgentPolicySO.id: - return managedAgentPolicySO; + case regularAgentPolicySO.id: + return regularAgentPolicySO; + case hostedAgentPolicySO.id: + return hostedAgentPolicySO; default: throw new Error('not found'); } @@ -267,12 +251,12 @@ function createClientMock() { // @ts-expect-error esClientMock.get.mockImplementation(async ({ id }) => { switch (id) { - case agentInManagedDoc._id: - return { body: agentInManagedDoc }; - case agentInUnmanagedDoc2._id: - return { body: agentInUnmanagedDoc2 }; - case agentInUnmanagedDoc._id: - return { body: agentInUnmanagedDoc }; + case agentInHostedDoc._id: + return { body: agentInHostedDoc }; + case agentInRegularDoc2._id: + return { body: agentInRegularDoc2 }; + case agentInRegularDoc._id: + return { body: agentInRegularDoc }; default: throw new Error('not found'); } @@ -287,12 +271,12 @@ function createClientMock() { // @ts-expect-error const docs = params?.body.docs.map(({ _id }) => { switch (_id) { - case agentInManagedDoc._id: - return agentInManagedDoc; - case agentInUnmanagedDoc2._id: - return agentInUnmanagedDoc2; - case agentInUnmanagedDoc._id: - return agentInUnmanagedDoc; + case agentInHostedDoc._id: + return agentInHostedDoc; + case agentInRegularDoc2._id: + return agentInRegularDoc2; + case agentInRegularDoc._id: + return agentInRegularDoc; default: throw new Error('not found'); } diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.ts index c97dc128de591..fc1f80fe7521b 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.ts @@ -29,7 +29,7 @@ async function unenrollAgentIsAllowed( const agentPolicy = await getAgentPolicyForAgent(soClient, esClient, agentId); if (agentPolicy?.is_managed) { throw new AgentUnenrollmentError( - `Cannot unenroll ${agentId} from a managed agent policy ${agentPolicy.id}` + `Cannot unenroll ${agentId} from a hosted agent policy ${agentPolicy.id}` ); } diff --git a/x-pack/plugins/fleet/server/services/agents/upgrade.ts b/x-pack/plugins/fleet/server/services/agents/upgrade.ts index c791a5b7c10ce..61e785828bf23 100644 --- a/x-pack/plugins/fleet/server/services/agents/upgrade.ts +++ b/x-pack/plugins/fleet/server/services/agents/upgrade.ts @@ -47,7 +47,7 @@ export async function sendUpgradeAgentAction({ const agentPolicy = await getAgentPolicyForAgent(soClient, esClient, agentId); if (agentPolicy?.is_managed) { throw new IngestManagerError( - `Cannot upgrade agent ${agentId} in managed policy ${agentPolicy.id}` + `Cannot upgrade agent ${agentId} in hosted agent policy ${agentPolicy.id}` ); } @@ -119,17 +119,17 @@ export async function sendUpgradeAgentsActions( const agentPolicies = await agentPolicyService.getByIDs(soClient, Array.from(policyIdsToGet), { fields: ['is_managed'], }); - const managedPolicies = agentPolicies.reduce>((acc, policy) => { + const hostedPolicies = agentPolicies.reduce>((acc, policy) => { acc[policy.id] = policy.is_managed; return acc; }, {}); - const isManagedAgent = (agent: Agent) => agent.policy_id && managedPolicies[agent.policy_id]; + const isHostedAgent = (agent: Agent) => agent.policy_id && hostedPolicies[agent.policy_id]; - // results from getAgents with options.kuery '' (or even 'active:false') may include managed agents + // results from getAgents with options.kuery '' (or even 'active:false') may include hosted agents // filter them out unless options.force const agentsToCheckUpgradeable = 'kuery' in options && !options.force - ? givenAgents.filter((agent: Agent) => !isManagedAgent(agent)) + ? givenAgents.filter((agent: Agent) => !isHostedAgent(agent)) : givenAgents; const kibanaVersion = appContextService.getKibanaVersion(); @@ -141,8 +141,10 @@ export async function sendUpgradeAgentsActions( throw new IngestManagerError(`${agent.id} is not upgradeable`); } - if (!options.force && isManagedAgent(agent)) { - throw new IngestManagerError(`Cannot upgrade agent in managed policy ${agent.policy_id}`); + if (!options.force && isHostedAgent(agent)) { + throw new IngestManagerError( + `Cannot upgrade agent in hosted agent policy ${agent.policy_id}` + ); } return agent; }) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 0857338469794..7c009299a3de3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -76,7 +76,7 @@ class PackagePolicyService { } if (parentAgentPolicy.is_managed && !options?.force) { throw new IngestManagerError( - `Cannot add integrations to managed policy ${parentAgentPolicy.id}` + `Cannot add integrations to hosted agent policy ${parentAgentPolicy.id}` ); } if ( diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f5b685a609b49..29f162a005a98 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -23479,4 +23479,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "フィールドを選択してください。", "xpack.watcher.watcherDescription": "アラートの作成、管理、監視によりデータへの変更を検知します。" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 88639fa7246c0..0553e3c195532 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -23847,4 +23847,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "此字段必填。", "xpack.watcher.watcherDescription": "通过创建、管理和监测警报来检测数据中的更改。" } -} \ No newline at end of file +} diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts index 8dcc3049ccd3a..779c4d767c000 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts @@ -327,37 +327,37 @@ export default function ({ getService }: FtrProviderContext) { after(async () => { await esArchiver.unload('fleet/empty_fleet_server'); }); - let managedPolicy: any | undefined; - it('should prevent managed policies being deleted', async () => { + let hostedPolicy: any | undefined; + it('should prevent hosted policies being deleted', async () => { const { body: { item: createdPolicy }, } = await supertest .post(`/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') .send({ - name: 'Managed policy', + name: 'Hosted policy', namespace: 'default', is_managed: true, }) .expect(200); - managedPolicy = createdPolicy; + hostedPolicy = createdPolicy; const { body } = await supertest .post('/api/fleet/agent_policies/delete') .set('kbn-xsrf', 'xxx') - .send({ agentPolicyId: managedPolicy.id }) + .send({ agentPolicyId: hostedPolicy.id }) .expect(400); - expect(body.message).to.contain('Cannot delete managed policy'); + expect(body.message).to.contain('Cannot delete hosted agent policy'); }); - it('should allow unmanaged policies being deleted', async () => { + it('should allow regular policies being deleted', async () => { const { - body: { item: unmanagedPolicy }, + body: { item: regularPolicy }, } = await supertest - .put(`/api/fleet/agent_policies/${managedPolicy.id}`) + .put(`/api/fleet/agent_policies/${hostedPolicy.id}`) .set('kbn-xsrf', 'xxxx') .send({ - name: 'Unmanaged policy', + name: 'Regular policy', namespace: 'default', is_managed: false, }) @@ -366,11 +366,11 @@ export default function ({ getService }: FtrProviderContext) { const { body } = await supertest .post('/api/fleet/agent_policies/delete') .set('kbn-xsrf', 'xxx') - .send({ agentPolicyId: unmanagedPolicy.id }); + .send({ agentPolicyId: regularPolicy.id }); expect(body).to.eql({ - id: unmanagedPolicy.id, - name: 'Unmanaged policy', + id: regularPolicy.id, + name: 'Regular policy', }); }); }); diff --git a/x-pack/test/fleet_api_integration/apis/agents/reassign.ts b/x-pack/test/fleet_api_integration/apis/agents/reassign.ts index 47bafd57ea3ad..ad3c224bb9236 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/reassign.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/reassign.ts @@ -55,8 +55,8 @@ export default function (providerContext: FtrProviderContext) { .expect(404); }); - it('can reassign from unmanaged policy to unmanaged', async () => { - // policy2 is not managed + it('can reassign from regular agent policy to regular', async () => { + // policy2 is not hosted // reassign succeeds await supertest .put(`/api/fleet/agents/agent1/reassign`) @@ -67,8 +67,8 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }); - it('cannot reassign from unmanaged policy to managed', async () => { - // agent1 is enrolled in policy1. set policy1 to managed + it('cannot reassign from regular agent policy to hosted', async () => { + // agent1 is enrolled in policy1. set policy1 to hosted await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') @@ -138,8 +138,8 @@ export default function (providerContext: FtrProviderContext) { expect(agent3data.body.item.policy_id).to.eql('policy2'); }); - it('should allow to reassign multiple agents by id -- mixed invalid, managed, etc', async () => { - // agent1 is enrolled in policy1. set policy1 to managed + it('should allow to reassign multiple agents by id -- mixed invalid, hosted, etc', async () => { + // agent1 is enrolled in policy1. set policy1 to hosted await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') @@ -157,7 +157,7 @@ export default function (providerContext: FtrProviderContext) { expect(body).to.eql({ agent2: { success: false, - error: 'Cannot reassign an agent from managed agent policy policy1', + error: 'Cannot reassign an agent from hosted agent policy policy1', }, INVALID_ID: { success: false, @@ -165,7 +165,7 @@ export default function (providerContext: FtrProviderContext) { }, agent3: { success: false, - error: 'Cannot reassign an agent from managed agent policy policy1', + error: 'Cannot reassign an agent from hosted agent policy policy1', }, }); diff --git a/x-pack/test/fleet_api_integration/apis/agents/unenroll.ts b/x-pack/test/fleet_api_integration/apis/agents/unenroll.ts index 60a588090048a..f0e41d75136c3 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/unenroll.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/unenroll.ts @@ -74,8 +74,8 @@ export default function (providerContext: FtrProviderContext) { await esArchiver.unload('fleet/empty_fleet_server'); }); - it('/agents/{agent_id}/unenroll should fail for managed policy', async () => { - // set policy to managed + it('/agents/{agent_id}/unenroll should fail for hosted agent policy', async () => { + // set policy to hosted await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') @@ -85,8 +85,8 @@ export default function (providerContext: FtrProviderContext) { await supertest.post(`/api/fleet/agents/agent1/unenroll`).set('kbn-xsrf', 'xxx').expect(400); }); - it('/agents/{agent_id}/unenroll should allow from unmanaged policy', async () => { - // set policy to unmanaged + it('/agents/{agent_id}/unenroll should allow from regular agent policy', async () => { + // set policy to regular await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') @@ -117,8 +117,8 @@ export default function (providerContext: FtrProviderContext) { expect(outputAPIKeys[0].invalidated).eql(true); }); - it('/agents/bulk_unenroll should not allow unenroll from managed policy', async () => { - // set policy to managed + it('/agents/bulk_unenroll should not allow unenroll from hosted agent policy', async () => { + // set policy to hosted await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') @@ -138,11 +138,11 @@ export default function (providerContext: FtrProviderContext) { expect(unenrolledBody).to.eql({ agent2: { success: false, - error: 'Cannot unenroll agent2 from a managed agent policy policy1', + error: 'Cannot unenroll agent2 from a hosted agent policy policy1', }, agent3: { success: false, - error: 'Cannot unenroll agent3 from a managed agent policy policy1', + error: 'Cannot unenroll agent3 from a hosted agent policy policy1', }, }); // but agents are still enrolled @@ -158,8 +158,8 @@ export default function (providerContext: FtrProviderContext) { expect(agent2data.body.item.active).to.eql(true); }); - it('/agents/bulk_unenroll should allow to unenroll multiple agents by id from an unmanaged policy', async () => { - // set policy to unmanaged + it('/agents/bulk_unenroll should allow to unenroll multiple agents by id from an regular agent policy', async () => { + // set policy to regular await supertest .put(`/api/fleet/agent_policies/policy1`) .set('kbn-xsrf', 'xxx') diff --git a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts index 545399134c79d..142c360e9232a 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/upgrade.ts @@ -210,8 +210,8 @@ export default function (providerContext: FtrProviderContext) { expect(res.body.message).to.equal('agent agent1 is not upgradeable'); }); - it('enrolled in a managed policy should respond 400 to upgrade and not update the agent SOs', async () => { - // update enrolled policy to managed + it('enrolled in a hosted agent policy should respond 400 to upgrade and not update the agent SOs', async () => { + // update enrolled policy to hosted await supertest.put(`/api/fleet/agent_policies/policy1`).set('kbn-xsrf', 'xxxx').send({ name: 'Test policy', namespace: 'default', @@ -229,13 +229,15 @@ export default function (providerContext: FtrProviderContext) { }, }, }); - // attempt to upgrade agent in managed policy + // attempt to upgrade agent in hosted agent policy const { body } = await supertest .post(`/api/fleet/agents/agent1/upgrade`) .set('kbn-xsrf', 'xxx') .send({ version: kibanaVersion }) .expect(400); - expect(body.message).to.contain('Cannot upgrade agent agent1 in managed policy policy1'); + expect(body.message).to.contain( + 'Cannot upgrade agent agent1 in hosted agent policy policy1' + ); const agent1data = await supertest.get(`/api/fleet/agents/agent1`); expect(typeof agent1data.body.item.upgrade_started_at).to.be('undefined'); @@ -543,12 +545,12 @@ export default function (providerContext: FtrProviderContext) { .expect(400); }); - it('enrolled in a managed policy bulk upgrade should respond with 200 and object of results. Should not update the managed agent SOs', async () => { - // move agent2 to policy2 to keep it unmanaged + it('enrolled in a hosted agent policy bulk upgrade should respond with 200 and object of results. Should not update the hosted agent SOs', async () => { + // move agent2 to policy2 to keep it regular await supertest.put(`/api/fleet/agents/agent2/reassign`).set('kbn-xsrf', 'xxx').send({ policy_id: 'policy2', }); - // update enrolled policy to managed + // update enrolled policy to hosted await supertest.put(`/api/fleet/agent_policies/policy1`).set('kbn-xsrf', 'xxxx').send({ name: 'Test policy', namespace: 'default', @@ -580,7 +582,7 @@ export default function (providerContext: FtrProviderContext) { }, }, }); - // attempt to upgrade agent in managed policy + // attempt to upgrade agent in hosted agent policy const { body } = await supertest .post(`/api/fleet/agents/bulk_upgrade`) .set('kbn-xsrf', 'xxx') @@ -591,7 +593,7 @@ export default function (providerContext: FtrProviderContext) { .expect(200); expect(body).to.eql({ - agent1: { success: false, error: 'Cannot upgrade agent in managed policy policy1' }, + agent1: { success: false, error: 'Cannot upgrade agent in hosted agent policy policy1' }, agent2: { success: true }, }); @@ -604,8 +606,8 @@ export default function (providerContext: FtrProviderContext) { expect(typeof agent2data.body.item.upgrade_started_at).to.be('string'); }); - it('enrolled in a managed policy bulk upgrade with force flag should respond with 200 and update the agent SOs', async () => { - // update enrolled policy to managed + it('enrolled in a hosted agent policy bulk upgrade with force flag should respond with 200 and update the agent SOs', async () => { + // update enrolled policy to hosted await supertest.put(`/api/fleet/agent_policies/policy1`).set('kbn-xsrf', 'xxxx').send({ name: 'Test policy', namespace: 'default', @@ -637,7 +639,7 @@ export default function (providerContext: FtrProviderContext) { }, }, }); - // attempt to upgrade agent in managed policy + // attempt to upgrade agent in hosted agent policy const { body } = await supertest .post(`/api/fleet/agents/bulk_upgrade`) .set('kbn-xsrf', 'xxx') diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts index e2e1cc2f584bb..27c5328b3ab08 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/create.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/create.ts @@ -46,20 +46,20 @@ export default function (providerContext: FtrProviderContext) { .send({ agentPolicyId }); }); - it('can only add to managed agent policies using the force parameter', async function () { - // get a managed policy + it('can only add to hosted agent policies using the force parameter', async function () { + // get a hosted policy const { - body: { item: managedPolicy }, + body: { item: hostedPolicy }, } = await supertest .post(`/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') .send({ - name: `Managed policy from ${Date.now()}`, + name: `Hosted policy from ${Date.now()}`, namespace: 'default', is_managed: true, }); - // try to add an integration to the managed policy + // try to add an integration to the hosted policy const { body: responseWithoutForce } = await supertest .post(`/api/fleet/package_policies`) .set('kbn-xsrf', 'xxxx') @@ -67,7 +67,7 @@ export default function (providerContext: FtrProviderContext) { name: 'filetest-1', description: '', namespace: 'default', - policy_id: managedPolicy.id, + policy_id: hostedPolicy.id, enabled: true, output_id: '', inputs: [], @@ -80,7 +80,9 @@ export default function (providerContext: FtrProviderContext) { .expect(400); expect(responseWithoutForce.statusCode).to.be(400); - expect(responseWithoutForce.message).to.contain('Cannot add integrations to managed policy'); + expect(responseWithoutForce.message).to.contain( + 'Cannot add integrations to hosted agent policy' + ); // try same request with `force: true` const { body: responseWithForce } = await supertest @@ -91,7 +93,7 @@ export default function (providerContext: FtrProviderContext) { name: 'filetest-1', description: '', namespace: 'default', - policy_id: managedPolicy.id, + policy_id: hostedPolicy.id, enabled: true, output_id: '', inputs: [], @@ -107,7 +109,7 @@ export default function (providerContext: FtrProviderContext) { // delete policy we just made await supertest.post(`/api/fleet/agent_policies/delete`).set('kbn-xsrf', 'xxxx').send({ - agentPolicyId: managedPolicy.id, + agentPolicyId: hostedPolicy.id, }); }); diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts b/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts index 15aba758c85d0..5889349f57fa0 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/delete.ts @@ -87,8 +87,8 @@ export default function (providerContext: FtrProviderContext) { await getService('esArchiver').unload('fleet/empty_fleet_server'); }); - it('should fail on managed agent policies', async function () { - // update existing policy to managed + it('should fail on hosted agent policies', async function () { + // update existing policy to hosted await supertest .put(`/api/fleet/agent_policies/${agentPolicy.id}`) .set('kbn-xsrf', 'xxxx') @@ -110,7 +110,9 @@ export default function (providerContext: FtrProviderContext) { expect(Array.isArray(results)); expect(results.length).to.be(1); expect(results[0].success).to.be(false); - expect(results[0].body.message).to.contain('Cannot remove integrations of managed policy'); + expect(results[0].body.message).to.contain( + 'Cannot remove integrations of hosted agent policy' + ); // same, but with force const { body: resultsWithForce } = await supertest @@ -124,7 +126,7 @@ export default function (providerContext: FtrProviderContext) { expect(resultsWithForce.length).to.be(1); expect(resultsWithForce[0].success).to.be(true); - // revert existing policy to unmanaged + // revert existing policy to regular await supertest .put(`/api/fleet/agent_policies/${agentPolicy.id}`) .set('kbn-xsrf', 'xxxx') @@ -136,7 +138,7 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }); - it('should work for unmanaged policies', async function () { + it('should work for regular policies', async function () { await supertest .post(`/api/fleet/package_policies/delete`) .set('kbn-xsrf', 'xxxx') diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts index 6e6a475cd4824..5a0ff90669def 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts @@ -46,7 +46,7 @@ export default function (providerContext: FtrProviderContext) { .post(`/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') .send({ - name: 'Test managed policy', + name: 'Test hosted agent policy', namespace: 'default', is_managed: true, }); From 540924b5be2544b2a5fd1ec90560dbcd2cd42c3a Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 15 Apr 2021 20:28:18 -0500 Subject: [PATCH 05/25] [Security Solution][Detections] Adds Nested CTI row renderer (#96275) * Move alert-specific mocks to more declarative mock file * Add placeholder interface for ECS threat fields * Test and implement CTI row renderer The display details are not yet implemented, but those will be fleshed out in the ThreatMatchRow component. * Pass full fields data to our row renderers This data is not used by any existing row renderers and so this commit is mostly just plumbing that data through. This is necessary, however, for our new threat match row renderer as it requires nested fields, which cannot be retrieved through the mechanism that retrieves the existing row renderer data. However, these nested fields are available, if requested, through this other data structure, hence this plumbing. For now to minimize changes I'm marking this as an optional field; however in reality a value will always be present. * Rewrite existing row renderer in terms of flattened data Updates logic, tests and mocks accordingly. * Moving logic into discrete files * helpers * explicit fields file, which will hopefully be part of the renderer API at some point * parent component to split data into "rows" as defined by our renderer * row component for stateless presentation of a single match * Register threat match row rendere Adds tentative copy, example row, and accompanying mock data. * WIP: Rendering draggable fields but hit the data loss issue with nested fields being flattened * WIP: implementing row renderer against new data format I haven't yet deleted the old (new?) unused path yet. Cleanup to come. * Updating based on new data * Rewrites isInstance logic for new data as helper, hasThreatMatchValue * Updating types and tests * Adds to the previously empty ThreatEcs * Revert "Pass full fields data to our row renderers" This reverts commit 19c93ee0732166747b5472433cd5fc813638e21b. We ended up extending the existing data (albeit from the fields response!). * Fix draggables * adds contextId and eventId to pass to draggable * We don't have a order-independent key for each individual ThreatMatchRow, due to matched.id not being mapped/returned in the fields response * Fixes up a few things related to using the new data format * Move indicator field strings to constants * Fix example data for CTI row renderer * Adds missing Threat ECS types * Move CTI field constants to common folder In order to use these in both the row renderer and the server request, we need to move them to common/ * Remove redundant CTI fields from client request These are currently hardcoded on the backend of the events/all query (via TIMELINE_EVENTS_FIELDS); declaring them on both ends is arguably confusing, and we're going with YAGNI for now. * Add missing graphQL type This was causing type errors as this enum exists both here and in common/, and I had only updated one of them. * Updates tests One is still failing due to an outdated test subject, but I expect this to change after an upcoming meeting so leaving it for now. * Split ThreatMatchRow into subcomponents One for displaying match details, and another for indicator details The indicator details will be sparse, so there's going to be some conditional rendering in there. * Make CTI row renderer look nice * Adds translations for copy * Fixes most of our layout woes with more flexbox! * Conditional rendering of indicator details based on data * tests * Make indicator reference field an external link Leverages the existing FormattedFieldValue component, with one minor tweak to add this field to the URL allowlist. * Back to consistent horizontal spacing, here The draggable badges are a little odd in that their full box isn't indicated until hover, making the visual weight a little off. * Add hr as a visual separator between each match "row" of the row renderer * Fix tests broken due to addition of a new row renderer These tests are all implicitly testing the list of row renderers. * Full-width hr At certain container widths, a half-width hr is not sufficient. * More descriptive constant Obviates the need for the accompanying comments. * More realistic data Also ensures less traffic to urlhaus ;) * Remove useless comment * Add threat_match row renderer type to GQL client Gennin' beanz * Ensure contextId is unique for each CTI subrow We need to add the row index to our contextId to ensure that our draggables work correctly for multiple rows, since each row will necessarily have the same eventId and timelineId. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../security_solution/common/cti/constants.ts | 34 ++++ .../security_solution/common/ecs/index.ts | 2 + .../common/ecs/rule/index.ts | 2 +- .../common/ecs/threat/index.ts | 25 +++ .../matrix_histogram/events/index.ts | 4 +- .../common/types/timeline/index.ts | 1 + .../public/common/mock/index.ts | 1 + .../common/mock/mock_detection_alerts.ts | 112 +++++++++++ .../public/common/mock/mock_ecs.ts | 66 ------- .../public/common/mock/mock_timeline_data.ts | 24 +++ .../public/graphql/introspection.json | 6 + .../security_solution/public/graphql/types.ts | 1 + .../row_renderers_browser/catalog/index.tsx | 8 + .../catalog/translations.ts | 13 ++ .../row_renderers_browser/examples/index.tsx | 1 + .../examples/threat_match.tsx | 23 +++ .../threat_match_row.test.tsx.snap | 33 ++++ .../threat_match_row_renderer.test.tsx.snap | 31 +++ .../timeline/body/renderers/cti/helpers.ts | 28 +++ .../body/renderers/cti/indicator_details.tsx | 117 +++++++++++ .../body/renderers/cti/match_details.tsx | 64 ++++++ .../renderers/cti/threat_match_row.test.tsx | 187 ++++++++++++++++++ .../body/renderers/cti/threat_match_row.tsx | 95 +++++++++ .../cti/threat_match_row_renderer.test.tsx | 65 ++++++ .../cti/threat_match_row_renderer.tsx | 17 ++ .../body/renderers/cti/threat_match_rows.tsx | 41 ++++ .../body/renderers/formatted_field.tsx | 8 +- .../timeline/body/renderers/index.ts | 2 + .../__snapshots__/index.test.tsx.snap | 5 + .../__snapshots__/index.test.tsx.snap | 5 + .../__snapshots__/index.test.tsx.snap | 5 + .../timelines/containers/index.test.tsx | 4 +- .../server/graphql/timeline/schema.gql.ts | 1 + .../security_solution/server/graphql/types.ts | 1 + .../timeline/factory/events/all/constants.ts | 11 +- 35 files changed, 963 insertions(+), 80 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/cti/constants.ts create mode 100644 x-pack/plugins/security_solution/common/ecs/threat/index.ts create mode 100644 x-pack/plugins/security_solution/public/common/mock/mock_detection_alerts.ts create mode 100644 x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/threat_match.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row.test.tsx.snap create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row_renderer.test.tsx.snap create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/indicator_details.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/match_details.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.test.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.test.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_rows.tsx diff --git a/x-pack/plugins/security_solution/common/cti/constants.ts b/x-pack/plugins/security_solution/common/cti/constants.ts new file mode 100644 index 0000000000000..cdd4a564f3d73 --- /dev/null +++ b/x-pack/plugins/security_solution/common/cti/constants.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { INDICATOR_DESTINATION_PATH } from '../constants'; + +export const MATCHED_ATOMIC = 'matched.atomic'; +export const MATCHED_FIELD = 'matched.field'; +export const MATCHED_TYPE = 'matched.type'; +export const INDICATOR_MATCH_SUBFIELDS = [MATCHED_ATOMIC, MATCHED_FIELD, MATCHED_TYPE]; + +export const INDICATOR_MATCHED_ATOMIC = `${INDICATOR_DESTINATION_PATH}.${MATCHED_ATOMIC}`; +export const INDICATOR_MATCHED_FIELD = `${INDICATOR_DESTINATION_PATH}.${MATCHED_FIELD}`; +export const INDICATOR_MATCHED_TYPE = `${INDICATOR_DESTINATION_PATH}.${MATCHED_TYPE}`; + +export const EVENT_DATASET = 'event.dataset'; +export const EVENT_REFERENCE = 'event.reference'; +export const PROVIDER = 'provider'; + +export const INDICATOR_DATASET = `${INDICATOR_DESTINATION_PATH}.${EVENT_DATASET}`; +export const INDICATOR_REFERENCE = `${INDICATOR_DESTINATION_PATH}.${EVENT_REFERENCE}`; +export const INDICATOR_PROVIDER = `${INDICATOR_DESTINATION_PATH}.${PROVIDER}`; + +export const CTI_ROW_RENDERER_FIELDS = [ + INDICATOR_MATCHED_ATOMIC, + INDICATOR_MATCHED_FIELD, + INDICATOR_MATCHED_TYPE, + INDICATOR_DATASET, + INDICATOR_REFERENCE, + INDICATOR_PROVIDER, +]; diff --git a/x-pack/plugins/security_solution/common/ecs/index.ts b/x-pack/plugins/security_solution/common/ecs/index.ts index 4c57f6419d5db..8054b3c8521db 100644 --- a/x-pack/plugins/security_solution/common/ecs/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/index.ts @@ -28,6 +28,7 @@ import { UserEcs } from './user'; import { WinlogEcs } from './winlog'; import { ProcessEcs } from './process'; import { SystemEcs } from './system'; +import { ThreatEcs } from './threat'; import { Ransomware } from './ransomware'; export interface Ecs { @@ -58,6 +59,7 @@ export interface Ecs { process?: ProcessEcs; file?: FileEcs; system?: SystemEcs; + threat?: ThreatEcs; // This should be temporary eql?: { parentId: string; sequenceNumber: string }; Ransomware?: Ransomware; diff --git a/x-pack/plugins/security_solution/common/ecs/rule/index.ts b/x-pack/plugins/security_solution/common/ecs/rule/index.ts index 5463b21f6b7f7..ae7e5064a8ece 100644 --- a/x-pack/plugins/security_solution/common/ecs/rule/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/rule/index.ts @@ -9,7 +9,7 @@ export interface RuleEcs { id?: string[]; rule_id?: string[]; name?: string[]; - false_positives: string[]; + false_positives?: string[]; saved_id?: string[]; timeline_id?: string[]; timeline_title?: string[]; diff --git a/x-pack/plugins/security_solution/common/ecs/threat/index.ts b/x-pack/plugins/security_solution/common/ecs/threat/index.ts new file mode 100644 index 0000000000000..19923a82dc846 --- /dev/null +++ b/x-pack/plugins/security_solution/common/ecs/threat/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventEcs } from '../event'; + +interface ThreatMatchEcs { + atomic?: string[]; + field?: string[]; + type?: string[]; +} + +export interface ThreatIndicatorEcs { + matched?: ThreatMatchEcs; + event?: EventEcs & { reference?: string[] }; + provider?: string[]; + type?: string[]; +} + +export interface ThreatEcs { + indicator: ThreatIndicatorEcs[]; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts index b2e0461b0b9b8..4df376acb256e 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts @@ -26,7 +26,9 @@ export interface EventsActionGroupData { doc_count: number; } -export type Fields = Record; +export interface Fields { + [x: string]: T | Array>; +} export interface EventHit extends SearchHit { sort: string[]; diff --git a/x-pack/plugins/security_solution/common/types/timeline/index.ts b/x-pack/plugins/security_solution/common/types/timeline/index.ts index 5fb7d1a74fc36..9def70048410a 100644 --- a/x-pack/plugins/security_solution/common/types/timeline/index.ts +++ b/x-pack/plugins/security_solution/common/types/timeline/index.ts @@ -206,6 +206,7 @@ export enum RowRendererId { system_fim = 'system_fim', system_security_event = 'system_security_event', system_socket = 'system_socket', + threat_match = 'threat_match', zeek = 'zeek', } diff --git a/x-pack/plugins/security_solution/public/common/mock/index.ts b/x-pack/plugins/security_solution/public/common/mock/index.ts index 469c3d9101eb4..ee34cc1798b54 100644 --- a/x-pack/plugins/security_solution/public/common/mock/index.ts +++ b/x-pack/plugins/security_solution/public/common/mock/index.ts @@ -10,6 +10,7 @@ export * from './header'; export * from './hook_wrapper'; export * from './index_pattern'; export * from './mock_detail_item'; +export * from './mock_detection_alerts'; export * from './mock_ecs'; export * from './mock_local_storage'; export * from './mock_timeline_data'; diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_detection_alerts.ts b/x-pack/plugins/security_solution/public/common/mock/mock_detection_alerts.ts new file mode 100644 index 0000000000000..2d93e7e0dc3a7 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/mock/mock_detection_alerts.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Ecs } from '../../../common/ecs'; +import { TimelineNonEcsData } from '../../../common/search_strategy'; + +export const mockEcsDataWithAlert: Ecs = { + _id: '1', + timestamp: '2018-11-05T19:03:25.937Z', + host: { + name: ['apache'], + ip: ['192.168.0.1'], + }, + event: { + id: ['1'], + action: ['Action'], + category: ['Access'], + module: ['nginx'], + severity: [3], + }, + source: { + ip: ['192.168.0.1'], + port: [80], + }, + destination: { + ip: ['192.168.0.3'], + port: [6343], + }, + user: { + id: ['1'], + name: ['john.dee'], + }, + geo: { + region_name: ['xx'], + country_iso_code: ['xx'], + }, + signal: { + rule: { + created_at: ['2020-01-10T21:11:45.839Z'], + updated_at: ['2020-01-10T21:11:45.839Z'], + created_by: ['elastic'], + description: ['24/7'], + enabled: [true], + false_positives: ['test-1'], + filters: [], + from: ['now-300s'], + id: ['b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea'], + immutable: [false], + index: ['auditbeat-*'], + interval: ['5m'], + rule_id: ['rule-id-1'], + language: ['kuery'], + output_index: ['.siem-signals-default'], + max_signals: [100], + risk_score: ['21'], + query: ['user.name: root or user.name: admin'], + references: ['www.test.co'], + saved_id: ["Garrett's IP"], + timeline_id: ['1234-2136-11ea-9864-ebc8cc1cb8c2'], + timeline_title: ['Untitled timeline'], + severity: ['low'], + updated_by: ['elastic'], + tags: [], + to: ['now'], + type: ['saved_query'], + threat: [], + note: ['# this is some markdown documentation'], + version: ['1'], + }, + }, +}; + +export const getDetectionAlertMock = (overrides: Partial = {}): Ecs => ({ + ...mockEcsDataWithAlert, + ...overrides, +}); + +export const getThreatMatchDetectionAlert = (overrides: Partial = {}): Ecs => ({ + ...mockEcsDataWithAlert, + signal: { + ...mockEcsDataWithAlert.signal, + rule: { + ...mockEcsDataWithAlert.rule, + name: ['mock threat_match rule'], + type: ['threat_match'], + }, + }, + threat: { + indicator: [ + { + matched: { + atomic: ['matched.atomic'], + field: ['matched.atomic'], + type: ['matched.domain'], + }, + }, + ], + }, + ...overrides, +}); + +export const getDetectionAlertFieldsMock = ( + fields: TimelineNonEcsData[] = [] +): TimelineNonEcsData[] => [ + { field: '@timestamp', value: ['2021-03-27T06:28:47.292Z'] }, + { field: 'signal.rule.type', value: ['threat_match'] }, + ...fields, +]; diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_ecs.ts b/x-pack/plugins/security_solution/public/common/mock/mock_ecs.ts index a28c2cc3bc581..f44c5c335cd21 100644 --- a/x-pack/plugins/security_solution/public/common/mock/mock_ecs.ts +++ b/x-pack/plugins/security_solution/public/common/mock/mock_ecs.ts @@ -1026,69 +1026,3 @@ export const mockEcsData: Ecs[] = [ }, }, ]; - -export const mockEcsDataWithAlert: Ecs = { - _id: '1', - timestamp: '2018-11-05T19:03:25.937Z', - host: { - name: ['apache'], - ip: ['192.168.0.1'], - }, - event: { - id: ['1'], - action: ['Action'], - category: ['Access'], - module: ['nginx'], - severity: [3], - }, - source: { - ip: ['192.168.0.1'], - port: [80], - }, - destination: { - ip: ['192.168.0.3'], - port: [6343], - }, - user: { - id: ['1'], - name: ['john.dee'], - }, - geo: { - region_name: ['xx'], - country_iso_code: ['xx'], - }, - signal: { - rule: { - created_at: ['2020-01-10T21:11:45.839Z'], - updated_at: ['2020-01-10T21:11:45.839Z'], - created_by: ['elastic'], - description: ['24/7'], - enabled: [true], - false_positives: ['test-1'], - filters: [], - from: ['now-300s'], - id: ['b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea'], - immutable: [false], - index: ['auditbeat-*'], - interval: ['5m'], - rule_id: ['rule-id-1'], - language: ['kuery'], - output_index: ['.siem-signals-default'], - max_signals: [100], - risk_score: ['21'], - query: ['user.name: root or user.name: admin'], - references: ['www.test.co'], - saved_id: ["Garrett's IP"], - timeline_id: ['1234-2136-11ea-9864-ebc8cc1cb8c2'], - timeline_title: ['Untitled timeline'], - severity: ['low'], - updated_by: ['elastic'], - tags: [], - to: ['now'], - type: ['saved_query'], - threat: [], - note: ['# this is some markdown documentation'], - version: ['1'], - }, - }, -}; diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_timeline_data.ts b/x-pack/plugins/security_solution/public/common/mock/mock_timeline_data.ts index f016b6cc34539..6a3c6468f43d5 100644 --- a/x-pack/plugins/security_solution/public/common/mock/mock_timeline_data.ts +++ b/x-pack/plugins/security_solution/public/common/mock/mock_timeline_data.ts @@ -1088,6 +1088,30 @@ export const mockTimelineData: TimelineItem[] = [ geo: { region_name: ['xx'], country_iso_code: ['xx'] }, }, }, + { + _id: '32', + data: [], + ecs: { + _id: 'BuBP4W0BOpWiDweSoYSg', + timestamp: '2019-10-18T23:59:15.091Z', + threat: { + indicator: [ + { + matched: { + atomic: ['192.168.1.1'], + field: ['source.ip'], + type: ['ip'], + }, + event: { + dataset: ['threatintel.example_dataset'], + reference: ['https://example.com'], + }, + provider: ['indicator_provider'], + }, + ], + }, + }, + }, ]; export const mockFimFileCreatedEvent: Ecs = { diff --git a/x-pack/plugins/security_solution/public/graphql/introspection.json b/x-pack/plugins/security_solution/public/graphql/introspection.json index 0a41ca05b8753..752173ded5163 100644 --- a/x-pack/plugins/security_solution/public/graphql/introspection.json +++ b/x-pack/plugins/security_solution/public/graphql/introspection.json @@ -1699,6 +1699,12 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "threat_match", + "description": "", + "isDeprecated": false, + "deprecationReason": null + }, { "name": "zeek", "description": "", "isDeprecated": false, "deprecationReason": null } ], "possibleTypes": null diff --git a/x-pack/plugins/security_solution/public/graphql/types.ts b/x-pack/plugins/security_solution/public/graphql/types.ts index 8ffd2995d0d97..a41111c3e123a 100644 --- a/x-pack/plugins/security_solution/public/graphql/types.ts +++ b/x-pack/plugins/security_solution/public/graphql/types.ts @@ -298,6 +298,7 @@ export enum RowRendererId { system_fim = 'system_fim', system_security_event = 'system_security_event', system_socket = 'system_socket', + threat_match = 'threat_match', zeek = 'zeek', } diff --git a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/index.tsx index 283a239acad24..f724c19913c8e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/index.tsx @@ -24,6 +24,7 @@ import { SystemFimExample, SystemSecurityEventExample, SystemSocketExample, + ThreatMatchExample, ZeekExample, } from '../examples'; import * as i18n from './translations'; @@ -204,6 +205,13 @@ export const renderers: RowRendererOption[] = [ example: SuricataExample, searchableDescription: `${i18n.SURICATA_DESCRIPTION_PART1} ${i18n.SURICATA_NAME} ${i18n.SURICATA_DESCRIPTION_PART2}`, }, + { + id: RowRendererId.threat_match, + name: i18n.THREAT_MATCH_NAME, + description: i18n.THREAT_MATCH_DESCRIPTION, + example: ThreatMatchExample, + searchableDescription: `${i18n.THREAT_MATCH_NAME} ${i18n.THREAT_MATCH_DESCRIPTION}`, + }, { id: RowRendererId.zeek, name: i18n.ZEEK_NAME, diff --git a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/translations.ts b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/translations.ts index a0d6d4e121891..95dce2e96d186 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/translations.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/catalog/translations.ts @@ -230,6 +230,19 @@ export const SYSTEM_DESCRIPTION_PART3 = i18n.translate( 'All datasets send both periodic state information (e.g. all currently running processes) and real-time changes (e.g. when a new process starts or stops).', } ); +export const THREAT_MATCH_NAME = i18n.translate( + 'xpack.securitySolution.eventRenderers.threatMatchName', + { + defaultMessage: 'Threat Indicator Match', + } +); + +export const THREAT_MATCH_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.eventRenderers.threatMatchDescription', + { + defaultMessage: 'Summarizes events that matched threat indicators', + } +); export const ZEEK_NAME = i18n.translate('xpack.securitySolution.eventRenderers.zeekName', { defaultMessage: 'Zeek (formerly Bro)', diff --git a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/index.tsx index 6932ca01835cc..da9d6923c2b76 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/index.tsx @@ -19,4 +19,5 @@ export * from './system_file'; export * from './system_fim'; export * from './system_security_event'; export * from './system_socket'; +export * from './threat_match'; export * from './zeek'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/threat_match.tsx b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/threat_match.tsx new file mode 100644 index 0000000000000..9d7e5d48315e3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/row_renderers_browser/examples/threat_match.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { mockTimelineData } from '../../../../common/mock/mock_timeline_data'; +import { threatMatchRowRenderer } from '../../timeline/body/renderers/cti/threat_match_row_renderer'; +import { ROW_RENDERER_BROWSER_EXAMPLE_TIMELINE_ID } from '../constants'; + +const ThreatMatchExampleComponent: React.FC = () => ( + <> + {threatMatchRowRenderer.renderRow({ + browserFields: {}, + data: mockTimelineData[31].ecs, + timelineId: ROW_RENDERER_BROWSER_EXAMPLE_TIMELINE_ID, + })} + +); +export const ThreatMatchExample = React.memo(ThreatMatchExampleComponent); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row.test.tsx.snap new file mode 100644 index 0000000000000..5e86ba25e4ba8 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row.test.tsx.snap @@ -0,0 +1,33 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ThreatMatchRowView matches the registered snapshot 1`] = ` + + + + + + + + +`; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row_renderer.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row_renderer.test.tsx.snap new file mode 100644 index 0000000000000..6e6dbddc6d9a0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/__snapshots__/threat_match_row_renderer.test.tsx.snap @@ -0,0 +1,31 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`threatMatchRowRenderer #renderRow renders correctly against snapshot 1`] = ` + + + + + + + +`; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts new file mode 100644 index 0000000000000..84dcef327736b --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/helpers.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isEmpty } from 'lodash'; +import styled from 'styled-components'; + +import { INDICATOR_DESTINATION_PATH } from '../../../../../../../common/constants'; +import { INDICATOR_MATCH_SUBFIELDS } from '../../../../../../../common/cti/constants'; +import { Ecs } from '../../../../../../../common/ecs'; +import { ThreatIndicatorEcs } from '../../../../../../../common/ecs/threat'; + +const getIndicatorEcs = (data: Ecs): ThreatIndicatorEcs[] => + get(data, INDICATOR_DESTINATION_PATH) ?? []; + +export const hasThreatMatchValue = (data: Ecs): boolean => + getIndicatorEcs(data).some((indicator) => + INDICATOR_MATCH_SUBFIELDS.some( + (indicatorMatchSubField) => !isEmpty(get(indicator, indicatorMatchSubField)) + ) + ); + +export const HorizontalSpacer = styled.div` + margin: 0 ${({ theme }) => theme.eui.paddingSizes.xs}; +`; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/indicator_details.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/indicator_details.tsx new file mode 100644 index 0000000000000..11846632f740e --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/indicator_details.tsx @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { + INDICATOR_DATASET, + INDICATOR_MATCHED_TYPE, + INDICATOR_PROVIDER, + INDICATOR_REFERENCE, +} from '../../../../../../../common/cti/constants'; +import { DraggableBadge } from '../../../../../../common/components/draggables'; +import { FormattedFieldValue } from '../formatted_field'; +import { HorizontalSpacer } from './helpers'; + +interface IndicatorDetailsProps { + contextId: string; + eventId: string; + indicatorDataset: string | undefined; + indicatorProvider: string | undefined; + indicatorReference: string | undefined; + indicatorType: string | undefined; +} + +export const IndicatorDetails: React.FC = ({ + contextId, + eventId, + indicatorDataset, + indicatorProvider, + indicatorReference, + indicatorType, +}) => ( + + {indicatorType && ( + + + + )} + {indicatorDataset && ( + <> + + + + + + + + + + )} + {indicatorProvider && ( + <> + + + + + + + + + + )} + {indicatorReference && ( + <> + + {':'} + + + + + + )} + +); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/match_details.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/match_details.tsx new file mode 100644 index 0000000000000..2195421301d31 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/match_details.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { INDICATOR_MATCHED_FIELD } from '../../../../../../../common/cti/constants'; +import { DraggableBadge } from '../../../../../../common/components/draggables'; +import { HorizontalSpacer } from './helpers'; + +interface MatchDetailsProps { + contextId: string; + eventId: string; + sourceField: string; + sourceValue: string; +} + +export const MatchDetails: React.FC = ({ + contextId, + eventId, + sourceField, + sourceValue, +}) => ( + + + + + + + + + + + + + +); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.test.tsx new file mode 100644 index 0000000000000..7f580642130fe --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.test.tsx @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { TestProviders } from '../../../../../../common/mock'; +import { useMountAppended } from '../../../../../../common/utils/use_mount_appended'; +import { ThreatMatchRowProps, ThreatMatchRowView } from './threat_match_row'; + +describe('ThreatMatchRowView', () => { + const mount = useMountAppended(); + + it('renders an indicator match row', () => { + const wrapper = shallow( + + ); + + expect(wrapper.find('[data-test-subj="threat-match-row"]').exists()).toEqual(true); + }); + + it('matches the registered snapshot', () => { + const wrapper = shallow( + + ); + + expect(wrapper).toMatchSnapshot(); + }); + + describe('field rendering', () => { + let baseProps: ThreatMatchRowProps; + const render = (props: ThreatMatchRowProps) => + mount( + + + + ); + + beforeEach(() => { + baseProps = { + contextId: 'contextId', + eventId: 'eventId', + indicatorDataset: 'dataset', + indicatorProvider: 'provider', + indicatorReference: 'http://example.com', + indicatorType: 'domain', + sourceField: 'host.name', + sourceValue: 'http://elastic.co', + }; + }); + + it('renders the match field', () => { + const wrapper = render(baseProps); + const matchField = wrapper.find('[data-test-subj="threat-match-details-source-field"]'); + expect(matchField.props()).toEqual( + expect.objectContaining({ + value: 'host.name', + }) + ); + }); + + it('renders the match value', () => { + const wrapper = render(baseProps); + const matchValue = wrapper.find('[data-test-subj="threat-match-details-source-value"]'); + expect(matchValue.props()).toEqual( + expect.objectContaining({ + field: 'host.name', + value: 'http://elastic.co', + }) + ); + }); + + it('renders the indicator type, if present', () => { + const wrapper = render(baseProps); + const indicatorType = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-type"]' + ); + expect(indicatorType.props()).toEqual( + expect.objectContaining({ + value: 'domain', + }) + ); + }); + + it('does not render the indicator type, if absent', () => { + const wrapper = render({ + ...baseProps, + indicatorType: undefined, + }); + const indicatorType = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-type"]' + ); + expect(indicatorType.exists()).toBeFalsy(); + }); + + it('renders the indicator dataset, if present', () => { + const wrapper = render(baseProps); + const indicatorDataset = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-dataset"]' + ); + expect(indicatorDataset.props()).toEqual( + expect.objectContaining({ + value: 'dataset', + }) + ); + }); + + it('does not render the indicator dataset, if absent', () => { + const wrapper = render({ + ...baseProps, + indicatorDataset: undefined, + }); + const indicatorDataset = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-dataset"]' + ); + expect(indicatorDataset.exists()).toBeFalsy(); + }); + + it('renders the indicator provider, if present', () => { + const wrapper = render(baseProps); + const indicatorProvider = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-provider"]' + ); + expect(indicatorProvider.props()).toEqual( + expect.objectContaining({ + value: 'provider', + }) + ); + }); + + it('does not render the indicator provider, if absent', () => { + const wrapper = render({ + ...baseProps, + indicatorProvider: undefined, + }); + const indicatorProvider = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-provider"]' + ); + expect(indicatorProvider.exists()).toBeFalsy(); + }); + + it('renders the indicator reference, if present', () => { + const wrapper = render(baseProps); + const indicatorReference = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-reference"]' + ); + expect(indicatorReference.props()).toEqual( + expect.objectContaining({ + value: 'http://example.com', + }) + ); + }); + + it('does not render the indicator reference, if absent', () => { + const wrapper = render({ + ...baseProps, + indicatorReference: undefined, + }); + const indicatorReference = wrapper.find( + '[data-test-subj="threat-match-indicator-details-indicator-reference"]' + ); + expect(indicatorReference.exists()).toBeFalsy(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.tsx new file mode 100644 index 0000000000000..ba5b0127df526 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash'; +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { Fields } from '../../../../../../../common/search_strategy'; +import { + EVENT_DATASET, + EVENT_REFERENCE, + MATCHED_ATOMIC, + MATCHED_FIELD, + MATCHED_TYPE, + PROVIDER, +} from '../../../../../../../common/cti/constants'; +import { MatchDetails } from './match_details'; +import { IndicatorDetails } from './indicator_details'; + +export interface ThreatMatchRowProps { + contextId: string; + eventId: string; + indicatorDataset: string | undefined; + indicatorProvider: string | undefined; + indicatorReference: string | undefined; + indicatorType: string | undefined; + sourceField: string; + sourceValue: string; +} + +export const ThreatMatchRow = ({ + contextId, + data, + eventId, +}: { + contextId: string; + data: Fields; + eventId: string; +}) => { + const props = { + contextId, + eventId, + indicatorDataset: get(data, EVENT_DATASET)[0] as string | undefined, + indicatorReference: get(data, EVENT_REFERENCE)[0] as string | undefined, + indicatorProvider: get(data, PROVIDER)[0] as string | undefined, + indicatorType: get(data, MATCHED_TYPE)[0] as string | undefined, + sourceField: get(data, MATCHED_FIELD)[0] as string, + sourceValue: get(data, MATCHED_ATOMIC)[0] as string, + }; + + return ; +}; + +export const ThreatMatchRowView = ({ + contextId, + eventId, + indicatorDataset, + indicatorProvider, + indicatorReference, + indicatorType, + sourceField, + sourceValue, +}: ThreatMatchRowProps) => { + return ( + + + + + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.test.tsx new file mode 100644 index 0000000000000..6687179e5b887 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.test.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { getThreatMatchDetectionAlert } from '../../../../../../common/mock'; + +import { threatMatchRowRenderer } from './threat_match_row_renderer'; + +describe('threatMatchRowRenderer', () => { + let threatMatchData: ReturnType; + + beforeEach(() => { + threatMatchData = getThreatMatchDetectionAlert(); + }); + + describe('#isInstance', () => { + it('is false for an empty event', () => { + const emptyEvent = { + _id: 'my_id', + '@timestamp': ['2020-11-17T14:48:08.922Z'], + }; + expect(threatMatchRowRenderer.isInstance(emptyEvent)).toBe(false); + }); + + it('is false for an alert with indicator data but no match', () => { + const indicatorTypeData = getThreatMatchDetectionAlert({ + threat: { + indicator: [{ type: ['url'] }], + }, + }); + expect(threatMatchRowRenderer.isInstance(indicatorTypeData)).toBe(false); + }); + + it('is false for an alert with threat match fields but no data', () => { + const emptyThreatMatchData = getThreatMatchDetectionAlert({ + threat: { + indicator: [{ matched: { type: [] } }], + }, + }); + expect(threatMatchRowRenderer.isInstance(emptyThreatMatchData)).toBe(false); + }); + + it('is true for an alert event with present indicator match fields', () => { + expect(threatMatchRowRenderer.isInstance(threatMatchData)).toBe(true); + }); + }); + + describe('#renderRow', () => { + it('renders correctly against snapshot', () => { + const children = threatMatchRowRenderer.renderRow({ + browserFields: {}, + data: threatMatchData, + timelineId: 'test', + }); + const wrapper = shallow({children}); + expect(wrapper).toMatchSnapshot(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.tsx new file mode 100644 index 0000000000000..2a7e8ce02d79f --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_row_renderer.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RowRendererId } from '../../../../../../../common/types/timeline'; +import { RowRenderer } from '../row_renderer'; +import { hasThreatMatchValue } from './helpers'; +import { ThreatMatchRows } from './threat_match_rows'; + +export const threatMatchRowRenderer: RowRenderer = { + id: RowRendererId.threat_match, + isInstance: hasThreatMatchValue, + renderRow: ThreatMatchRows, +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_rows.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_rows.tsx new file mode 100644 index 0000000000000..cc34f9e63b5e2 --- /dev/null +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/cti/threat_match_rows.tsx @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiHorizontalRule } from '@elastic/eui'; +import { get } from 'lodash'; +import React, { Fragment } from 'react'; +import styled from 'styled-components'; + +import { Fields } from '../../../../../../../common/search_strategy'; +import { ID_FIELD_NAME } from '../../../../../../common/components/event_details/event_id'; +import { RowRenderer, RowRendererContainer } from '../row_renderer'; +import { ThreatMatchRow } from './threat_match_row'; + +const SpacedContainer = styled.div` + margin: ${({ theme }) => theme.eui.paddingSizes.s} 0; +`; + +export const ThreatMatchRows: RowRenderer['renderRow'] = ({ data, timelineId }) => { + const indicators = get(data, 'threat.indicator') as Fields[]; + const eventId = get(data, ID_FIELD_NAME); + + return ( + + + {indicators.map((indicator, index) => { + const contextId = `threat-match-row-${timelineId}-${eventId}-${index}`; + return ( + + + {index < indicators.length - 1 && } + + ); + })} + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx index e227c87b99870..12effcd3fa81f 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/formatted_field.tsx @@ -9,6 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui'; import { isNumber, isEmpty } from 'lodash/fp'; import React from 'react'; +import { INDICATOR_REFERENCE } from '../../../../../../common/cti/constants'; import { DefaultDraggable } from '../../../../../common/components/draggables'; import { Bytes, BYTES_FORMAT } from './bytes'; import { Duration, EVENT_DURATION_FIELD_NAME } from '../../../duration'; @@ -116,7 +117,12 @@ const FormattedFieldValueComponent: React.FC<{ ); } else if ( - [RULE_REFERENCE_FIELD_NAME, REFERENCE_URL_FIELD_NAME, EVENT_URL_FIELD_NAME].includes(fieldName) + [ + RULE_REFERENCE_FIELD_NAME, + REFERENCE_URL_FIELD_NAME, + EVENT_URL_FIELD_NAME, + INDICATOR_REFERENCE, + ].includes(fieldName) ) { return renderUrl({ contextId, eventId, fieldName, linkValue, truncate, value }); } else if (columnNamesNotDraggable.includes(fieldName)) { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/index.ts b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/index.ts index 209a9414f62f1..537a24bbfd953 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/index.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/renderers/index.ts @@ -15,6 +15,7 @@ import { suricataRowRenderer } from './suricata/suricata_row_renderer'; import { unknownColumnRenderer } from './unknown_column_renderer'; import { zeekRowRenderer } from './zeek/zeek_row_renderer'; import { systemRowRenderers } from './system/generic_row_renderer'; +import { threatMatchRowRenderer } from './cti/threat_match_row_renderer'; // The row renderers are order dependent and will return the first renderer // which returns true from its isInstance call. The bottom renderers which @@ -24,6 +25,7 @@ import { systemRowRenderers } from './system/generic_row_renderer'; // plainRowRenderer always returns true to everything which is why it always // should be last. export const defaultRowRenderers: RowRenderer[] = [ + threatMatchRowRenderer, ...auditdRowRenderers, ...systemRowRenderers, suricataRowRenderer, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/eql_tab_content/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/eql_tab_content/__snapshots__/index.test.tsx.snap index 7d237ecaf92df..9ec1fa7071277 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/eql_tab_content/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/eql_tab_content/__snapshots__/index.test.tsx.snap @@ -143,6 +143,11 @@ In other use cases the message field can be used to concatenate different values renderCellValue={[Function]} rowRenderers={ Array [ + Object { + "id": "threat_match", + "isInstance": [Function], + "renderRow": [Function], + }, Object { "id": "auditd", "isInstance": [Function], diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/__snapshots__/index.test.tsx.snap index ef73ba9f24db3..ce59d191a472d 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/pinned_tab_content/__snapshots__/index.test.tsx.snap @@ -138,6 +138,11 @@ In other use cases the message field can be used to concatenate different values renderCellValue={[Function]} rowRenderers={ Array [ + Object { + "id": "threat_match", + "isInstance": [Function], + "renderRow": [Function], + }, Object { "id": "auditd", "isInstance": [Function], diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap index 46c85f634ff6b..f6ff6b50221b7 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_tab_content/__snapshots__/index.test.tsx.snap @@ -279,6 +279,11 @@ In other use cases the message field can be used to concatenate different values renderCellValue={[Function]} rowRenderers={ Array [ + Object { + "id": "threat_match", + "isInstance": [Function], + "renderRow": [Function], + }, Object { "id": "auditd", "isInstance": [Function], diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx index ddcecb885a8ff..496107e910d76 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx @@ -159,7 +159,7 @@ describe('useTimelineEvents', () => { loadPage: result.current[1].loadPage, pageInfo: result.current[1].pageInfo, refetch: result.current[1].refetch, - totalCount: 31, + totalCount: 32, updatedAt: result.current[1].updatedAt, }, ]); @@ -202,7 +202,7 @@ describe('useTimelineEvents', () => { loadPage: result.current[1].loadPage, pageInfo: result.current[1].pageInfo, refetch: result.current[1].refetch, - totalCount: 31, + totalCount: 32, updatedAt: result.current[1].updatedAt, }, ]); diff --git a/x-pack/plugins/security_solution/server/graphql/timeline/schema.gql.ts b/x-pack/plugins/security_solution/server/graphql/timeline/schema.gql.ts index 05a824e3630bd..98e7103e61224 100644 --- a/x-pack/plugins/security_solution/server/graphql/timeline/schema.gql.ts +++ b/x-pack/plugins/security_solution/server/graphql/timeline/schema.gql.ts @@ -171,6 +171,7 @@ export const timelineSchema = gql` system_fim system_security_event system_socket + threat_match zeek } diff --git a/x-pack/plugins/security_solution/server/graphql/types.ts b/x-pack/plugins/security_solution/server/graphql/types.ts index 29d366e20c299..a60a6dd6093d1 100644 --- a/x-pack/plugins/security_solution/server/graphql/types.ts +++ b/x-pack/plugins/security_solution/server/graphql/types.ts @@ -300,6 +300,7 @@ export enum RowRendererId { system_fim = 'system_fim', system_security_event = 'system_security_event', system_socket = 'system_socket', + threat_match = 'threat_match', zeek = 'zeek', } diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts index 29b0df9e4bbf7..38188a1616bfc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts @@ -5,14 +5,7 @@ * 2.0. */ -export const TIMELINE_CTI_FIELDS = [ - 'threat.indicator.event.dataset', - 'threat.indicator.event.reference', - 'threat.indicator.matched.atomic', - 'threat.indicator.matched.field', - 'threat.indicator.matched.type', - 'threat.indicator.provider', -]; +import { CTI_ROW_RENDERER_FIELDS } from '../../../../../../common/cti/constants'; export const TIMELINE_EVENTS_FIELDS = [ '@timestamp', @@ -239,5 +232,5 @@ export const TIMELINE_EVENTS_FIELDS = [ 'zeek.ssl.established', 'zeek.ssl.resumed', 'zeek.ssl.version', - ...TIMELINE_CTI_FIELDS, + ...CTI_ROW_RENDERER_FIELDS, ]; From b5ae056ac4e75e99c1171977ca1395c250c78b16 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 15 Apr 2021 21:27:43 -0500 Subject: [PATCH 06/25] [Security Solution][Detections] ML Rules accept multiple ML Job IDs (#97073) * Adds helper to normalize legacy ML rule field to an array This will be used on read of rules, to normalize legacy rules while avoiding an explicit migration. * Fix our detection-specific ML search function Luckily this was just a translation layer to our anomaly call, and the underlying functions already accepted an array of strings. * WIP: Run rules against multiple ML Job IDs We don't yet support creation of rules with multiple job ids, either on the API or the UI, but when we do they will work. Note: the logic was previously to generate an error if the underlying job was not running, but to still query and generate alerts. Extending that logic to multiple jobs: if any are not running, we generate an error but continue querying and generating alerts. * WIP: updating ml rule schemas to support multiple job IDs * Simplify normalization method We don't care about null or empty string values here; those were holdovers from copying the logic of normalizeThreshold and don't apply to this situation. * Move normalized types to separate file to fix circular dependency Our use of NonEmptyArray within common/schemas seemed to be causing the above; this fixes it for now. * Normalize ML job_ids param at the API layer Previous changes to the base types already covered the majority of routes; this updates the miscellaneous helpers that don't leverage those shared utilities. At the DB level, the forthcoming migration will ensure that we always have "normalized" job IDs as an array. * Count stopped ML Jobs as partial failure during ML Rule execution Since we continue to query anomalies and potentially generate alerts, a "failure" status is no longer the most accurate for this situation. * Update 7.13 alerts migration to allow multi-job ML Rules This ensures that we can assume string[] for this field during rule execution. * Display N job statuses on rule details * WIP: converts MLJobSelect to a multiselect Unfortunately, the SuperSelect does not allow multiselect so we need to convert this to a combobox. Luckily we can reuse most of the code here and remain relatively clean. Since all combobox options must be the same (fixed) height, we're somewhat more limited than before for displaying the rows. The truncation appears fine, but I need to figure out a way to display the full description as well. * Update client-side logic to handle an array of ML job_ids * Marginally more legible error message * Conditionally call our normalize helper only if we have a value This fixes a type error where TS could not infer that the return value would not be undefined despite knowing that the argument was never undefined. I tried some fancy conditional generic types, but that didn't work. This is more analogous to normalizeThresholdObject now, anyway. * Fix remaining type error * Clean up our ML executor tests with existing contract mocks * Update ML Executor tests with new logic We now record a partial failure instead of an error. * Add and update tests for new ML normalization logic * Add and update integration tests for ML Rules Ensures that dealing with legacy job formats continues to work in the API. * Fix a type error These params can no longer be strings. * Update ML cypress test to create a rule with 2 ML jobs If we can create a rule with 2 jobs, we should also be able to create a rule with 1 job. * Remove unused constant * Persist a partial failure message written by a rule executor We added the result.warning field as a way to indicate that a partial failure was written to the rule, but neglected to account for that in the main rule execution code, which caused a success status to immediately overwrite the partial failure if the rule execution did not otherwise fail/short-circuit. --- .../server/saved_objects/migrations.test.ts | 52 ++++++++++++ .../server/saved_objects/migrations.ts | 6 ++ .../schemas/common/schemas.ts | 3 +- .../schemas/types/normalized_ml_job_id.ts | 22 +++++ .../common/detection_engine/utils.test.ts | 18 ++++ .../common/detection_engine/utils.ts | 3 + .../machine_learning_rule.spec.ts | 6 +- .../security_solution/cypress/objects/rule.ts | 4 +- .../cypress/screens/create_new_rule.ts | 5 +- .../cypress/tasks/create_new_rule.ts | 13 +-- .../exceptions/add_exception_modal/index.tsx | 5 +- .../exceptions/edit_exception_modal/index.tsx | 5 +- .../rules/description_step/index.tsx | 6 +- .../description_step/ml_job_description.tsx | 12 ++- .../components/rules/ml_job_select/index.tsx | 67 ++++++++------- .../rules/step_define_rule/index.tsx | 2 +- .../rules/step_define_rule/translations.tsx | 2 +- .../detection_engine/rules/types.ts | 2 +- .../rules/all/__mocks__/mock.ts | 2 +- .../rules/create/helpers.test.ts | 4 +- .../detection_engine/rules/helpers.test.tsx | 8 +- .../pages/detection_engine/rules/helpers.tsx | 2 +- .../pages/detection_engine/rules/types.ts | 4 +- .../rules/patch_rules_bulk_route.test.ts | 2 +- .../routes/rules/patch_rules_route.test.ts | 2 +- .../routes/rules/utils.test.ts | 4 +- .../rules/create_rules.test.ts | 22 ++++- .../detection_engine/rules/create_rules.ts | 9 +- .../rules/patch_rules.test.ts | 27 +++++- .../lib/detection_engine/rules/patch_rules.ts | 9 +- .../schemas/rule_converters.ts | 7 +- .../schemas/rule_schemas.mock.ts | 2 +- .../detection_engine/schemas/rule_schemas.ts | 4 +- .../signals/executors/ml.test.ts | 82 ++++++------------- .../detection_engine/signals/executors/ml.ts | 28 ++++--- .../signals/find_ml_signals.ts | 6 +- .../signals/rule_status_service.mock.ts | 36 ++------ .../signals/signal_rule_alert_type.ts | 2 +- .../security_and_spaces/tests/create_rules.ts | 15 ++++ .../security_and_spaces/tests/patch_rules.ts | 16 ++++ .../security_and_spaces/tests/update_rules.ts | 22 +++++ .../detection_engine_api_integration/utils.ts | 6 +- 42 files changed, 365 insertions(+), 189 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/schemas/types/normalized_ml_job_id.ts diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 4df75ab60b496..a080809bbc968 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -973,6 +973,58 @@ describe('7.13.0', () => { }, }); }); + + test('security solution ML alert with string in machineLearningJobId is converted to an array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + anomalyThreshold: 20, + machineLearningJobId: 'my_job_id', + }, + }); + + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); + }); + + test('security solution ML alert with an array in machineLearningJobId is preserved', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id', 'my_other_job_id'], + }, + }); + + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id', 'my_other_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); + }); }); function getUpdatedAt(): string { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index 8ebeb401b313c..c9327ed8f186a 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -400,6 +400,12 @@ function removeNullsFromSecurityRules( ? params.lists : [], threatFilters: convertNullToUndefined(params.threatFilters), + machineLearningJobId: + params.machineLearningJobId == null + ? undefined + : Array.isArray(params.machineLearningJobId) + ? params.machineLearningJobId + : [params.machineLearningJobId], }, }, }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index c61ab85f43270..b5f88aa144814 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -22,6 +22,7 @@ import { PositiveIntegerGreaterThanZero } from '../types/positive_integer_greate import { PositiveInteger } from '../types/positive_integer'; import { NonEmptyString } from '../types/non_empty_string'; import { parseScheduleDates } from '../../parse_schedule_dates'; +import { machine_learning_job_id_normalized } from '../types/normalized_ml_job_id'; export const author = t.array(t.string); export type Author = t.TypeOf; @@ -230,7 +231,7 @@ export type AnomalyThreshold = t.TypeOf; export const anomalyThresholdOrUndefined = t.union([anomaly_threshold, t.undefined]); export type AnomalyThresholdOrUndefined = t.TypeOf; -export const machine_learning_job_id = t.string; +export const machine_learning_job_id = t.union([t.string, machine_learning_job_id_normalized]); export type MachineLearningJobId = t.TypeOf; export const machineLearningJobIdOrUndefined = t.union([machine_learning_job_id, t.undefined]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/normalized_ml_job_id.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/normalized_ml_job_id.ts new file mode 100644 index 0000000000000..c826bce92c8a0 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/normalized_ml_job_id.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import * as t from 'io-ts'; + +import { NonEmptyArray } from './non_empty_array'; + +export const machine_learning_job_id_normalized = NonEmptyArray(t.string); +export type MachineLearningJobIdNormalized = t.TypeOf; + +export const machineLearningJobIdNormalizedOrUndefined = t.union([ + machine_learning_job_id_normalized, + t.undefined, +]); +export type MachineLearningJobIdNormalizedOrUndefined = t.TypeOf< + typeof machineLearningJobIdNormalizedOrUndefined +>; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 9377255dc85d5..c477036a07d85 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -10,6 +10,7 @@ import { hasLargeValueList, hasNestedEntry, isThreatMatchRule, + normalizeMachineLearningJobIds, normalizeThresholdField, } from './utils'; import { EntriesArray } from '../shared_imports'; @@ -175,3 +176,20 @@ describe('normalizeThresholdField', () => { expect(normalizeThresholdField('')).toEqual([]); }); }); + +describe('normalizeMachineLearningJobIds', () => { + it('converts a string to a string array', () => { + expect(normalizeMachineLearningJobIds('ml_job_id')).toEqual(['ml_job_id']); + }); + + it('preserves a single-valued array ', () => { + expect(normalizeMachineLearningJobIds(['ml_job_id'])).toEqual(['ml_job_id']); + }); + + it('preserves a multi-valued array ', () => { + expect(normalizeMachineLearningJobIds(['ml_job_id', 'other_ml_job_id'])).toEqual([ + 'ml_job_id', + 'other_ml_job_id', + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 1f4e4e140ce18..a8e0ffcccef82 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -62,5 +62,8 @@ export const normalizeThresholdObject = (threshold: Threshold): ThresholdNormali }; }; +export const normalizeMachineLearningJobIds = (value: string | string[]): string[] => + Array.isArray(value) ? value : [value]; + export const getRuleStatusText = (value: JobStatus | null | undefined): JobStatus | null => value === 'partial failure' ? 'warning' : value != null ? value : null; diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts index e420b970ad85f..6b88246cf5fb6 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/machine_learning_rule.spec.ts @@ -129,8 +129,10 @@ describe('Detection rules, machine learning', () => { ); getDetails(RULE_TYPE_DETAILS).should('have.text', 'Machine Learning'); getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - cy.get(MACHINE_LEARNING_JOB_STATUS).should('have.text', 'Stopped'); - cy.get(MACHINE_LEARNING_JOB_ID).should('have.text', machineLearningRule.machineLearningJob); + machineLearningRule.machineLearningJobs.forEach((machineLearningJob, jobIndex) => { + cy.get(MACHINE_LEARNING_JOB_STATUS).eq(jobIndex).should('have.text', 'Stopped'); + cy.get(MACHINE_LEARNING_JOB_ID).eq(jobIndex).should('have.text', machineLearningJob); + }); }); cy.get(SCHEDULE_DETAILS).within(() => { getDetails(RUNS_EVERY_DETAILS).should( diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index e85b3f45b4ea6..f083cc5da6f53 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -78,7 +78,7 @@ export interface ThreatIndicatorRule extends CustomRule { } export interface MachineLearningRule { - machineLearningJob: string; + machineLearningJobs: string[]; anomalyScoreThreshold: string; name: string; description: string; @@ -244,7 +244,7 @@ export const newThresholdRule: ThresholdRule = { }; export const machineLearningRule: MachineLearningRule = { - machineLearningJob: 'linux_anomalous_network_service', + machineLearningJobs: ['linux_anomalous_network_service', 'linux_anomalous_network_activity_ecs'], anomalyScoreThreshold: '20', name: 'New ML Rule Test', description: 'The new ML rule description.', diff --git a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts index b2b7e434348b4..8b9d9b144910d 100644 --- a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts @@ -108,9 +108,10 @@ export const LOOK_BACK_INTERVAL = export const LOOK_BACK_TIME_TYPE = '[data-test-subj="detectionEngineStepScheduleRuleFrom"] [data-test-subj="timeType"]'; -export const MACHINE_LEARNING_DROPDOWN = '[data-test-subj="mlJobSelect"] button'; +export const MACHINE_LEARNING_DROPDOWN_INPUT = + '[data-test-subj="mlJobSelect"] [data-test-subj="comboBoxInput"]'; -export const MACHINE_LEARNING_LIST = '.euiContextMenuItem__text'; +export const MACHINE_LEARNING_DROPDOWN_ITEM = '.euiFilterSelectItem'; export const MACHINE_LEARNING_TYPE = '[data-test-subj="machineLearningRuleType"]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts index 0c663a95a4bda..2b7308757f9f4 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts @@ -44,8 +44,7 @@ import { INVESTIGATION_NOTES_TEXTAREA, LOOK_BACK_INTERVAL, LOOK_BACK_TIME_TYPE, - MACHINE_LEARNING_DROPDOWN, - MACHINE_LEARNING_LIST, + MACHINE_LEARNING_DROPDOWN_INPUT, MACHINE_LEARNING_TYPE, MITRE_ATTACK_ADD_SUBTECHNIQUE_BUTTON, MITRE_ATTACK_ADD_TACTIC_BUTTON, @@ -86,6 +85,7 @@ import { THRESHOLD_FIELD_SELECTION, THRESHOLD_INPUT_AREA, THRESHOLD_TYPE, + MACHINE_LEARNING_DROPDOWN_ITEM, } from '../screens/create_new_rule'; import { TOAST_ERROR } from '../screens/shared'; import { SERVER_SIDE_EVENT_COUNT } from '../screens/timeline'; @@ -434,14 +434,17 @@ export const fillDefineIndicatorMatchRuleAndContinue = (rule: ThreatIndicatorRul }; export const fillDefineMachineLearningRuleAndContinue = (rule: MachineLearningRule) => { - cy.get(MACHINE_LEARNING_DROPDOWN).click({ force: true }); - cy.contains(MACHINE_LEARNING_LIST, rule.machineLearningJob).click(); + rule.machineLearningJobs.forEach((machineLearningJob) => { + cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).click({ force: true }); + cy.contains(MACHINE_LEARNING_DROPDOWN_ITEM, machineLearningJob).click(); + cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).type('{esc}'); + }); cy.get(ANOMALY_THRESHOLD_INPUT).type(`{selectall}${machineLearningRule.anomalyScoreThreshold}`, { force: true, }); getDefineContinueButton().should('exist').click({ force: true }); - cy.get(MACHINE_LEARNING_DROPDOWN).should('not.exist'); + cy.get(MACHINE_LEARNING_DROPDOWN_INPUT).should('not.exist'); }; export const goToDefineStepTab = () => { diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx index 07dcb2272748f..8b7fe572b9d24 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx @@ -140,10 +140,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({ memoSignalIndexName ); - const memoMlJobIds = useMemo( - () => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []), - [maybeRule] - ); + const memoMlJobIds = useMemo(() => maybeRule?.machine_learning_job_id ?? [], [maybeRule]); const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds); const memoRuleIndices = useMemo(() => { diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx index 2c996c600261b..5ad3baabedb6a 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx @@ -123,10 +123,7 @@ export const EditExceptionModal = memo(function EditExceptionModal({ memoSignalIndexName ); - const memoMlJobIds = useMemo( - () => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []), - [maybeRule] - ); + const memoMlJobIds = useMemo(() => maybeRule?.machine_learning_job_id ?? [], [maybeRule]); const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds); const memoRuleIndices = useMemo(() => { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx index 4e330f7c0bd07..9c40853794743 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/index.tsx @@ -36,7 +36,7 @@ import { buildThresholdDescription, buildThreatMappingDescription, } from './helpers'; -import { buildMlJobDescription } from './ml_job_description'; +import { buildMlJobsDescription } from './ml_job_description'; import { buildActionsDescription } from './actions_description'; import { buildThrottleDescription } from './throttle_description'; import { Threats, Type } from '../../../../../common/detection_engine/schemas/common/schemas'; @@ -74,8 +74,8 @@ export const StepRuleDescriptionComponent = ({ if (key === 'machineLearningJobId') { return [ ...acc, - buildMlJobDescription( - get(key, data) as string, + buildMlJobsDescription( + get(key, data) as string[], (get(key, schema) as { label: string }).label ), ]; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/ml_job_description.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/ml_job_description.tsx index d430a964ed79f..27afe847f7612 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/description_step/ml_job_description.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/description_step/ml_job_description.tsx @@ -104,7 +104,15 @@ const MlJobDescriptionComponent: React.FC<{ jobId: string }> = ({ jobId }) => { export const MlJobDescription = React.memo(MlJobDescriptionComponent); -export const buildMlJobDescription = (jobId: string, label: string): ListItems => ({ +const MlJobsDescription: React.FC<{ jobIds: string[] }> = ({ jobIds }) => ( + <> + {jobIds.map((jobId) => ( + + ))} + +); + +export const buildMlJobsDescription = (jobIds: string[], label: string): ListItems => ({ title: label, - description: , + description: , }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/ml_job_select/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/ml_job_select/index.tsx index cffdeeb491f2a..e5521492d3b5e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/ml_job_select/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/ml_job_select/index.tsx @@ -8,12 +8,13 @@ import React, { useCallback, useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { + EuiComboBox, + EuiComboBoxOptionOption, EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiIcon, EuiLink, - EuiSuperSelect, EuiText, } from '@elastic/eui'; @@ -27,6 +28,13 @@ import { ENABLE_ML_JOB_WARNING, } from '../step_define_rule/translations'; +interface MlJobValue { + id: string; + description: string; +} + +type MlJobOption = EuiComboBoxOptionOption; + const HelpTextWarningContainer = styled.div` margin-top: 10px; `; @@ -65,9 +73,9 @@ const HelpText: React.FC<{ href: string; showEnableWarning: boolean }> = ({ ); -const JobDisplay: React.FC<{ title: string; description: string }> = ({ title, description }) => ( +const JobDisplay: React.FC = ({ id, description }) => ( <> - {title} + {id}

{description}

 @@ -79,45 +87,44 @@ interface MlJobSelectProps { field: FieldHook; } +const renderJobOption = (option: MlJobOption) => ( + +); + export const MlJobSelect: React.FC = ({ describedByIds = [], field }) => { - const jobId = field.value as string; + const jobIds = field.value as string[]; const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field); const { loading, jobs } = useSecurityJobs(false); const mlUrl = useKibana().services.application.getUrlForApp('ml'); - const handleJobChange = useCallback( - (machineLearningJobId: string) => { - field.setValue(machineLearningJobId); + const handleJobSelect = useCallback( + (selectedJobOptions: MlJobOption[]): void => { + const selectedJobIds = selectedJobOptions.map((option) => option.value!.id); + field.setValue(selectedJobIds); }, [field] ); - const placeholderOption = { - value: 'placeholder', - inputDisplay: ML_JOB_SELECT_PLACEHOLDER_TEXT, - dropdownDisplay: ML_JOB_SELECT_PLACEHOLDER_TEXT, - disabled: true, - }; const jobOptions = jobs.map((job) => ({ - value: job.id, - inputDisplay: job.id, - dropdownDisplay: , + value: { + id: job.id, + description: job.description, + }, + label: job.id, })); - const options = [placeholderOption, ...jobOptions]; + const selectedJobOptions = jobOptions.filter((option) => jobIds.includes(option.value.id)); - const isJobRunning = useMemo(() => { - // If the selected job is not found in the list, it means the placeholder is selected - // and so we don't want to show the warning, thus isJobRunning will be true when 'job == null' - const job = jobs.find(({ id }) => id === jobId); - return job == null || isJobStarted(job.jobState, job.datafeedState); - }, [jobs, jobId]); + const allJobsRunning = useMemo(() => { + const selectedJobs = jobs.filter(({ id }) => jobIds.includes(id)); + return selectedJobs.every((job) => isJobStarted(job.jobState, job.datafeedState)); + }, [jobs, jobIds]); return ( } + helpText={} isInvalid={isInvalid} error={errorMessage} data-test-subj="mlJobSelect" @@ -125,12 +132,14 @@ export const MlJobSelect: React.FC = ({ describedByIds = [], f > - diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 362dbb4bb722b..29342bd32298e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -65,7 +65,7 @@ interface StepDefineRuleProps extends RuleStepProps { const stepDefineDefaultValue: DefineStepRule = { anomalyThreshold: 50, index: [], - machineLearningJobId: '', + machineLearningJobId: [], ruleType: 'query', threatIndex: [], queryBar: { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index 8d83854f9250c..273c8cf28a18a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -68,7 +68,7 @@ export const ENABLE_ML_JOB_WARNING = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.mlEnableJobWarningTitle', { defaultMessage: - 'This ML job is not currently running. Please set this job to run via "ML job settings" before activating this rule.', + 'One or more selected ML jobs are not currently running. Please set these job(s) to run via "ML job settings" before activating this rule.', } ); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts index b14297e34bd3e..2c3d6484aebdd 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts @@ -123,7 +123,7 @@ export const RuleSchema = t.intersection([ last_success_message: t.string, last_success_at: t.string, meta: MetaRule, - machine_learning_job_id: t.string, + machine_learning_job_id: t.array(t.string), output_index: t.string, query: t.string, rule_name_override, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index ee2c2c48d22ee..821413b361701 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -185,7 +185,7 @@ export const mockActionsStepRule = (enabled = false): ActionsStepRule => ({ export const mockDefineStepRule = (): DefineStepRule => ({ ruleType: 'query', anomalyThreshold: 50, - machineLearningJobId: '', + machineLearningJobId: [], index: ['filebeat-'], queryBar: mockQueryBar, threatQueryBar: mockQueryBar, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts index fdb0513d7b708..98d3dadc7bbcb 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.test.ts @@ -249,14 +249,14 @@ describe('helpers', () => { ...mockData, ruleType: 'machine_learning', anomalyThreshold: 44, - machineLearningJobId: 'some_jobert_id', + machineLearningJobId: ['some_jobert_id'], }; const result = formatDefineStepData(mockStepData); const expected: DefineStepRuleJson = { type: 'machine_learning', anomaly_threshold: 44, - machine_learning_job_id: 'some_jobert_id', + machine_learning_job_id: ['some_jobert_id'], timeline_id: '86aa74d0-2136-11ea-9864-ebc8cc1cb8c2', timeline_title: 'Titled timeline', }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 9c2e7751753ee..4c3e5b18d4c1b 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -34,7 +34,7 @@ import { import { getThreatMock } from '../../../../../common/detection_engine/schemas/types/threat.mock'; describe('rule helpers', () => { - // @ts-ignore + // @ts-expect-error moment.suppressDeprecationWarnings = true; describe('getStepsData', () => { test('returns object with about, define, schedule and actions step properties formatted', () => { @@ -51,7 +51,7 @@ describe('rule helpers', () => { ruleType: 'saved_query', anomalyThreshold: 50, index: ['auditbeat-*'], - machineLearningJobId: '', + machineLearningJobId: [], queryBar: { query: { query: 'user.name: root or user.name: admin', @@ -204,7 +204,7 @@ describe('rule helpers', () => { const expected = { ruleType: 'saved_query', anomalyThreshold: 50, - machineLearningJobId: '', + machineLearningJobId: [], index: ['auditbeat-*'], queryBar: { query: { @@ -246,7 +246,7 @@ describe('rule helpers', () => { const expected = { ruleType: 'saved_query', anomalyThreshold: 50, - machineLearningJobId: '', + machineLearningJobId: [], index: ['auditbeat-*'], queryBar: { query: { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 9bc3ab9103b42..03688264bcf46 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -81,7 +81,7 @@ export const getActionsStepsData = ( export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ ruleType: rule.type, anomalyThreshold: rule.anomaly_threshold ?? 50, - machineLearningJobId: rule.machine_learning_job_id ?? '', + machineLearningJobId: rule.machine_learning_job_id ?? [], index: rule.index ?? [], threatIndex: rule.threat_index ?? [], threatQueryBar: { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 8eb26073e52d2..58994c5a5f556 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -126,7 +126,7 @@ export interface AboutStepRiskScore { export interface DefineStepRule { anomalyThreshold: number; index: string[]; - machineLearningJobId: string; + machineLearningJobId: string[]; queryBar: FieldValueQueryBar; ruleType: Type; timeline: FieldValueTimeline; @@ -153,7 +153,7 @@ export interface DefineStepRuleJson { anomaly_threshold?: number; index?: string[]; filters?: Filter[]; - machine_learning_job_id?: string; + machine_learning_job_id?: string[]; saved_id?: string; query?: string; language?: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts index b83dad92d43b5..b6dd8a3fe0431 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts @@ -76,7 +76,7 @@ describe('patch_rules_bulk', () => { data: expect.objectContaining({ params: expect.objectContaining({ anomalyThreshold: 4, - machineLearningJobId: 'some_job_id', + machineLearningJobId: ['some_job_id'], }), }), }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts index 2fa72ae2a097e..9920ec5229a02 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts @@ -105,7 +105,7 @@ describe('patch_rules', () => { data: expect.objectContaining({ params: expect.objectContaining({ anomalyThreshold: 4, - machineLearningJobId: 'some_job_id', + machineLearningJobId: ['some_job_id'], }), }), }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts index ffa699daf9c95..b841507bc7a6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts @@ -87,14 +87,14 @@ describe('utils', () => { test('transforms ML Rule fields', () => { const mlRule = getAlertMock(getMlRuleParams()); mlRule.params.anomalyThreshold = 55; - mlRule.params.machineLearningJobId = 'some_job_id'; + mlRule.params.machineLearningJobId = ['some_job_id']; mlRule.params.type = 'machine_learning'; const rule = transformAlertToRule(mlRule); expect(rule).toEqual( expect.objectContaining({ anomaly_threshold: 55, - machine_learning_job_id: 'some_job_id', + machine_learning_job_id: ['some_job_id'], type: 'machine_learning', }) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.test.ts index bf114518533bc..c719412d27e4d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.test.ts @@ -9,7 +9,7 @@ import { createRules } from './create_rules'; import { getCreateMlRulesOptionsMock } from './create_rules.mock'; describe('createRules', () => { - it('calls the alertsClient with ML params', async () => { + it('calls the alertsClient with legacy ML params', async () => { const ruleOptions = getCreateMlRulesOptionsMock(); await createRules(ruleOptions); expect(ruleOptions.alertsClient.create).toHaveBeenCalledWith( @@ -17,7 +17,25 @@ describe('createRules', () => { data: expect.objectContaining({ params: expect.objectContaining({ anomalyThreshold: 55, - machineLearningJobId: 'new_job_id', + machineLearningJobId: ['new_job_id'], + }), + }), + }) + ); + }); + + it('calls the alertsClient with ML params', async () => { + const ruleOptions = { + ...getCreateMlRulesOptionsMock(), + machineLearningJobId: ['new_job_1', 'new_job_2'], + }; + await createRules(ruleOptions); + expect(ruleOptions.alertsClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + anomalyThreshold: 55, + machineLearningJobId: ['new_job_1', 'new_job_2'], }), }), }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 2a3d83f4baca7..db039bbc31390 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -5,7 +5,10 @@ * 2.0. */ -import { normalizeThresholdObject } from '../../../../common/detection_engine/utils'; +import { + normalizeMachineLearningJobIds, + normalizeThresholdObject, +} from '../../../../common/detection_engine/utils'; import { transformRuleToAlertAction } from '../../../../common/detection_engine/transform_actions'; import { SanitizedAlert } from '../../../../../alerting/common'; import { SERVER_APP_ID, SIGNALS_ID } from '../../../../common/constants'; @@ -89,7 +92,9 @@ export const createRules = async ({ timelineId, timelineTitle, meta, - machineLearningJobId, + machineLearningJobId: machineLearningJobId + ? normalizeMachineLearningJobIds(machineLearningJobId) + : undefined, filters, maxSignals, riskScore, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.test.ts index 65466b46f8d5e..e275a02b2b0c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.test.ts @@ -41,7 +41,7 @@ describe('patchRules', () => { ); }); - it('calls the alertsClient with ML params', async () => { + it('calls the alertsClient with legacy ML params', async () => { const rulesOptionsMock = getPatchMlRulesOptionsMock(); const ruleOptions: PatchRulesOptions = { ...rulesOptionsMock, @@ -56,7 +56,30 @@ describe('patchRules', () => { data: expect.objectContaining({ params: expect.objectContaining({ anomalyThreshold: 55, - machineLearningJobId: 'new_job_id', + machineLearningJobId: ['new_job_id'], + }), + }), + }) + ); + }); + + it('calls the alertsClient with new ML params', async () => { + const rulesOptionsMock = getPatchMlRulesOptionsMock(); + const ruleOptions: PatchRulesOptions = { + ...rulesOptionsMock, + machineLearningJobId: ['new_job_1', 'new_job_2'], + enabled: true, + }; + if (ruleOptions.rule != null) { + ruleOptions.rule.enabled = false; + } + await patchRules(ruleOptions); + expect(ruleOptions.alertsClient.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + params: expect.objectContaining({ + anomalyThreshold: 55, + machineLearningJobId: ['new_job_1', 'new_job_2'], }), }), }) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index bf769e46ab7bd..bccd1f2fb73ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -14,7 +14,10 @@ import { addTags } from './add_tags'; import { calculateVersion, calculateName, calculateInterval, removeUndefined } from './utils'; import { ruleStatusSavedObjectsClientFactory } from '../signals/rule_status_saved_objects_client'; import { internalRuleUpdate, RuleParams } from '../schemas/rule_schemas'; -import { normalizeThresholdObject } from '../../../../common/detection_engine/utils'; +import { + normalizeMachineLearningJobIds, + normalizeThresholdObject, +} from '../../../../common/detection_engine/utils'; class PatchError extends Error { public readonly statusCode: number; @@ -167,7 +170,9 @@ export const patchRules = async ({ version: calculatedVersion, exceptionsList, anomalyThreshold, - machineLearningJobId, + machineLearningJobId: machineLearningJobId + ? normalizeMachineLearningJobIds(machineLearningJobId) + : undefined, } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts index 65cf1d2f723c6..ee7ecaadfd95c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts @@ -7,7 +7,10 @@ import uuid from 'uuid'; import { SavedObject } from 'kibana/server'; -import { normalizeThresholdObject } from '../../../../common/detection_engine/utils'; +import { + normalizeMachineLearningJobIds, + normalizeThresholdObject, +} from '../../../../common/detection_engine/utils'; import { InternalRuleCreate, RuleParams, @@ -103,7 +106,7 @@ export const typeSpecificSnakeToCamel = (params: CreateTypeSpecific): TypeSpecif return { type: params.type, anomalyThreshold: params.anomaly_threshold, - machineLearningJobId: params.machine_learning_job_id, + machineLearningJobId: normalizeMachineLearningJobIds(params.machine_learning_job_id), }; } default: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts index 8c5825325bd2e..846a4e26410a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts @@ -88,7 +88,7 @@ export const getMlRuleParams = (): MachineLearningRuleParams => { ...getBaseRuleParams(), type: 'machine_learning', anomalyThreshold: 42, - machineLearningJobId: 'my-job', + machineLearningJobId: ['my-job'], }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts index cd2b5d0b9eda7..79b862d6419c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts @@ -36,7 +36,6 @@ import { query, queryOrUndefined, filtersOrUndefined, - machine_learning_job_id, max_signals, risk_score, risk_score_mapping, @@ -62,6 +61,7 @@ import { updated_at, } from '../../../../common/detection_engine/schemas/common/schemas'; import { SIGNALS_ID, SERVER_APP_ID } from '../../../../common/constants'; +import { machine_learning_job_id_normalized } from '../../../../common/detection_engine/schemas/types/normalized_ml_job_id'; const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); export const baseRuleParams = t.exact( @@ -167,7 +167,7 @@ export type ThresholdRuleParams = t.TypeOf; const machineLearningSpecificRuleParams = t.type({ type: t.literal('machine_learning'), anomalyThreshold: anomaly_threshold, - machineLearningJobId: machine_learning_job_id, + machineLearningJobId: machine_learning_job_id_normalized, }); export const machineLearningRuleParams = t.intersection([ baseRuleParams, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index a3db2e5cbfd99..e157750a7d51b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -15,52 +15,21 @@ import { buildRuleMessageFactory } from '../rule_messages'; import { getListClientMock } from '../../../../../../lists/server/services/lists/list_client.mock'; import { findMlSignals } from '../find_ml_signals'; import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; +import { mlPluginServerMock } from '../../../../../../ml/server/mocks'; +import { sampleRuleSO } from '../__mocks__/es_results'; +import { getRuleStatusServiceMock } from '../rule_status_service.mock'; jest.mock('../find_ml_signals'); jest.mock('../bulk_create_ml_signals'); describe('ml_executor', () => { - const jobsSummaryMock = jest.fn(); - const mlMock = { - mlClient: { - callAsInternalUser: jest.fn(), - close: jest.fn(), - asScoped: jest.fn(), - }, - jobServiceProvider: jest.fn().mockReturnValue({ - jobsSummary: jobsSummaryMock, - }), - anomalyDetectorsProvider: jest.fn(), - mlSystemProvider: jest.fn(), - modulesProvider: jest.fn(), - resultsServiceProvider: jest.fn(), - alertingServiceProvider: jest.fn(), - }; + let jobsSummaryMock: jest.Mock; + let mlMock: ReturnType; + let ruleStatusService: ReturnType; const exceptionItems = [getExceptionListItemSchemaMock()]; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleStatusService: Record; - const mlSO = { - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - type: 'alert', - version: '1', - updated_at: '2020-03-27T22:55:59.577Z', - attributes: { - actions: [], - enabled: true, - name: 'rule-name', - tags: ['some fake tag 1', 'some fake tag 2'], - createdBy: 'sample user', - createdAt: '2020-03-27T22:55:59.577Z', - updatedBy: 'sample user', - schedule: { - interval: '5m', - }, - throttle: 'no_actions', - params: getMlRuleParams(), - }, - references: [], - }; + const mlSO = sampleRuleSO(getMlRuleParams()); const buildRuleMessage = buildRuleMessageFactory({ id: mlSO.id, ruleId: mlSO.attributes.params.ruleId, @@ -69,15 +38,14 @@ describe('ml_executor', () => { }); beforeEach(() => { + jobsSummaryMock = jest.fn(); alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleStatusService = { - success: jest.fn(), - find: jest.fn(), - goingToRun: jest.fn(), - error: jest.fn(), - partialFailure: jest.fn(), - }; + mlMock = mlPluginServerMock.createSetupContract(); + mlMock.jobServiceProvider.mockReturnValue({ + jobsSummary: jobsSummaryMock, + }); + ruleStatusService = getRuleStatusServiceMock(); (findMlSignals as jest.Mock).mockResolvedValue({ _shards: {}, hits: { @@ -98,7 +66,7 @@ describe('ml_executor', () => { rule: mlSO, ml: undefined, exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + ruleStatusService, services: alertServices, logger, refresh: false, @@ -108,13 +76,13 @@ describe('ml_executor', () => { ).rejects.toThrow('ML plugin unavailable during rule execution'); }); - it('should throw an error if Machine learning job summary was null', async () => { + it('should record a partial failure if Machine learning job summary was null', async () => { jobsSummaryMock.mockResolvedValue([]); await mlExecutor({ rule: mlSO, ml: mlMock, exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, + ruleStatusService, services: alertServices, logger, refresh: false, @@ -122,14 +90,14 @@ describe('ml_executor', () => { listClient: getListClientMock(), }); expect(logger.warn).toHaveBeenCalled(); - expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); - expect(ruleStatusService.error).toHaveBeenCalled(); - expect(ruleStatusService.error.mock.calls[0][0]).toContain( - 'Machine learning job is not started' + expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); + expect(ruleStatusService.partialFailure).toHaveBeenCalled(); + expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( + 'Machine learning job(s) are not started' ); }); - it('should log an error if Machine learning job was not started', async () => { + it('should record a partial failure if Machine learning job was not started', async () => { jobsSummaryMock.mockResolvedValue([ { id: 'some_job_id', @@ -150,10 +118,10 @@ describe('ml_executor', () => { listClient: getListClientMock(), }); expect(logger.warn).toHaveBeenCalled(); - expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job is not started'); - expect(ruleStatusService.error).toHaveBeenCalled(); - expect(ruleStatusService.error.mock.calls[0][0]).toContain( - 'Machine learning job is not started' + expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); + expect(ruleStatusService.partialFailure).toHaveBeenCalled(); + expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( + 'Machine learning job(s) are not started' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index 338ad2dbe9d40..928767e922d67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -58,20 +58,28 @@ export const mlExecutor = async ({ const fakeRequest = {} as KibanaRequest; const summaryJobs = await ml .jobServiceProvider(fakeRequest, services.savedObjectsClient) - .jobsSummary([ruleParams.machineLearningJobId]); - const jobSummary = summaryJobs.find((job) => job.id === ruleParams.machineLearningJobId); + .jobsSummary(ruleParams.machineLearningJobId); + const jobSummaries = summaryJobs.filter((job) => + ruleParams.machineLearningJobId.includes(job.id) + ); - if (jobSummary == null || !isJobStarted(jobSummary.jobState, jobSummary.datafeedState)) { + if ( + jobSummaries.length < 1 || + jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) + ) { const errorMessage = buildRuleMessage( - 'Machine learning job is not started:', - `job id: "${ruleParams.machineLearningJobId}"`, - `job status: "${jobSummary?.jobState}"`, - `datafeed status: "${jobSummary?.datafeedState}"` + 'Machine learning job(s) are not started:', + ...jobSummaries.map((job) => + [ + `job id: "${job.id}"`, + `job status: "${job.jobState}"`, + `datafeed status: "${job.datafeedState}"`, + ].join(', ') + ) ); logger.warn(errorMessage); result.warning = true; - // TODO: change this to partialFailure since we don't immediately exit rule function and still do actions at the end? - await ruleStatusService.error(errorMessage); + await ruleStatusService.partialFailure(errorMessage); } const anomalyResults = await findMlSignals({ @@ -80,7 +88,7 @@ export const mlExecutor = async ({ // currently unused by the mlAnomalySearch function. request: ({} as unknown) as KibanaRequest, savedObjectsClient: services.savedObjectsClient, - jobId: ruleParams.machineLearningJobId, + jobIds: ruleParams.machineLearningJobId, anomalyThreshold: ruleParams.anomalyThreshold, from: ruleParams.from, to: ruleParams.to, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts index 12fe32e15d734..6870ae2d80bbf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts @@ -16,7 +16,7 @@ export const findMlSignals = async ({ ml, request, savedObjectsClient, - jobId, + jobIds, anomalyThreshold, from, to, @@ -25,7 +25,7 @@ export const findMlSignals = async ({ ml: MlPluginSetup; request: KibanaRequest; savedObjectsClient: SavedObjectsClientContract; - jobId: string; + jobIds: string[]; anomalyThreshold: number; from: string; to: string; @@ -33,7 +33,7 @@ export const findMlSignals = async ({ }): Promise => { const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient); const params = { - jobIds: [jobId], + jobIds, threshold: anomalyThreshold, earliestMs: dateMath.parse(from)?.valueOf() ?? 0, latestMs: dateMath.parse(to)?.valueOf() ?? 0, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.mock.ts index 04f2b6ff799da..1ecdf09880873 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.mock.ts @@ -5,37 +5,11 @@ * 2.0. */ -import { SavedObjectsClientContract } from '../../../../../../../src/core/server'; -import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; import { RuleStatusService } from './rule_status_service'; -export type RuleStatusServiceMock = jest.Mocked; - -export const ruleStatusServiceFactoryMock = async ({ - alertId, - ruleStatusClient, -}: { - alertId: string; - ruleStatusClient: RuleStatusSavedObjectsClient; -}): Promise => { - return { - goingToRun: jest.fn(), - - success: jest.fn(), - - partialFailure: jest.fn(), - - error: jest.fn(), - }; -}; - -export type RuleStatusSavedObjectsClientMock = jest.Mocked; - -export const ruleStatusSavedObjectsClientFactory = ( - savedObjectsClient: SavedObjectsClientContract -): RuleStatusSavedObjectsClientMock => ({ - find: jest.fn(), - create: jest.fn(), - update: jest.fn(), - delete: jest.fn(), +export const getRuleStatusServiceMock = (): jest.Mocked => ({ + goingToRun: jest.fn(), + success: jest.fn(), + partialFailure: jest.fn(), + error: jest.fn(), }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 419141d98d15a..637826a943480 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -330,7 +330,7 @@ export const signalRulesAlertType = ({ `[+] Finished indexing ${result.createdSignalsCount} signals into ${outputIndex}` ) ); - if (!hasError && !wroteWarningStatus) { + if (!hasError && !wroteWarningStatus && !result.warning) { await ruleStatusService.success('succeeded', { bulkCreateTimeDurations: result.bulkCreateTimes, searchAfterTimeDurations: result.searchAfterTimes, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts index 29eb84cddcb0b..5ec3374598776 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts @@ -224,6 +224,21 @@ export default ({ getService }: FtrProviderContext) => { expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId()); }); + it('creates a single Machine Learning rule from a legacy ML Rule format', async () => { + const legacyMlRule = { + ...getSimpleMlRule(), + machine_learning_job_id: 'some_job_id', + }; + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(legacyMlRule) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(getSimpleMlRuleOutput()); + }); + it('should create a single Machine Learning rule', async () => { const { body } = await supertest .post(DETECTION_ENGINE_RULES_URL) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/patch_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/patch_rules.ts index bc42ef92e6b2c..d20eb0492bbc4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/patch_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/patch_rules.ts @@ -55,6 +55,22 @@ export default ({ getService }: FtrProviderContext) => { expect(bodyToCompare).to.eql(outputRule); }); + it("should patch a machine_learning rule's job ID if in a legacy format", async () => { + await createRule(supertest, getSimpleMlRule('rule-1')); + + // patch a simple rule's name + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send({ rule_id: 'rule-1', machine_learning_job_id: 'some_job_id' }) + .expect(200); + + const outputRule = getSimpleMlRuleOutput(); + outputRule.version = 2; + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputRule); + }); + it('should patch a single rule property of name using a rule_id of type "machine learning"', async () => { await createRule(supertest, getSimpleMlRule('rule-1')); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/update_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/update_rules.ts index eebf4305a3ac1..5a4a04f71b3d5 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/update_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/update_rules.ts @@ -62,6 +62,28 @@ export default ({ getService }: FtrProviderContext) => { expect(bodyToCompare).to.eql(outputRule); }); + it("should update a rule's machine learning job ID if given a legacy job ID format", async () => { + await createRule(supertest, getSimpleMlRule('rule-1')); + + // update rule's machine_learning_job_id + const updatedRule = getSimpleMlRuleUpdate('rule-1'); + // @ts-expect-error updatedRule is the full union type here and thus is not narrowed to our ML params + updatedRule.machine_learning_job_id = 'legacy_job_id'; + delete updatedRule.id; + + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(updatedRule) + .expect(200); + + const outputRule = getSimpleMlRuleOutput(); + outputRule.machine_learning_job_id = ['legacy_job_id']; + outputRule.version = 2; + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputRule); + }); + it('should update a single rule property of name using a rule_id with a machine learning job', async () => { await createRule(supertest, getSimpleMlRule('rule-1')); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index a9c128ee87703..d821b57faf225 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -172,7 +172,7 @@ export const getSimpleMlRule = (ruleId = 'rule-1', enabled = false): CreateRules risk_score: 1, rule_id: ruleId, severity: 'high', - machine_learning_job_id: 'some_job_id', + machine_learning_job_id: ['some_job_id'], type: 'machine_learning', }); @@ -189,7 +189,7 @@ export const getSimpleMlRuleUpdate = (ruleId = 'rule-1', enabled = false): Updat risk_score: 1, rule_id: ruleId, severity: 'high', - machine_learning_job_id: 'some_job_id', + machine_learning_job_id: ['some_job_id'], type: 'machine_learning', }); @@ -344,7 +344,7 @@ export const getSimpleMlRuleOutput = (ruleId = 'rule-1'): Partial = name: 'Simple ML Rule', description: 'Simple Machine Learning Rule', anomaly_threshold: 44, - machine_learning_job_id: 'some_job_id', + machine_learning_job_id: ['some_job_id'], type: 'machine_learning', }; }; From 347882f01a3accfc01bd714dc4de4e8adbf52f26 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 15 Apr 2021 22:05:14 -0700 Subject: [PATCH 07/25] skip flaky suite (#91360) --- x-pack/test/api_integration/apis/security_solution/tls.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security_solution/tls.ts b/x-pack/test/api_integration/apis/security_solution/tls.ts index eadf7d2aac7ae..a8e0517e6ccdb 100644 --- a/x-pack/test/api_integration/apis/security_solution/tls.ts +++ b/x-pack/test/api_integration/apis/security_solution/tls.ts @@ -84,7 +84,8 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); - describe('Tls Test with Packetbeat', () => { + // Failing: See https://github.com/elastic/kibana/issues/91360 + describe.skip('Tls Test with Packetbeat', () => { describe('Tls Test', () => { before(() => esArchiver.load('packetbeat/tls')); after(() => esArchiver.unload('packetbeat/tls')); From 9987e3d73b8632f0e4c70b52286ea7be0353e555 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 15 Apr 2021 22:12:23 -0700 Subject: [PATCH 08/25] [kbnClient] fix basePath handling and export reponse type (#97277) Co-authored-by: spalger Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../src/kbn_client/kbn_client_import_export.ts | 1 + .../src/kbn_client/kbn_client_requester.ts | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts index bb5b99fdc4439..7f4d0160923bf 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_import_export.ts @@ -115,6 +115,7 @@ export class KbnClientImportExport { excludeExportDetails: true, includeReferencesDeep: true, }, + responseType: 'text', }); if (typeof resp.data !== 'string') { diff --git a/packages/kbn-test/src/kbn_client/kbn_client_requester.ts b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts index 2e1575aee1897..31cd3a6899568 100644 --- a/packages/kbn-test/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-test/src/kbn_client/kbn_client_requester.ts @@ -10,7 +10,7 @@ import Url from 'url'; import Https from 'https'; import Qs from 'querystring'; -import Axios, { AxiosResponse } from 'axios'; +import Axios, { AxiosResponse, ResponseType } from 'axios'; import { ToolingLog, isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils'; const isConcliftOnGetError = (error: any) => { @@ -53,6 +53,7 @@ export interface ReqOptions { body?: any; retries?: number; headers?: Record; + responseType?: ResponseType; } const delay = (ms: number) => @@ -84,11 +85,16 @@ export class KbnClientRequester { } public resolveUrl(relativeUrl: string = '/') { - return Url.resolve(this.pickUrl(), relativeUrl); + let baseUrl = this.pickUrl(); + if (!baseUrl.endsWith('/')) { + baseUrl += '/'; + } + const relative = relativeUrl.startsWith('/') ? relativeUrl.slice(1) : relativeUrl; + return Url.resolve(baseUrl, relative); } async request(options: ReqOptions): Promise> { - const url = Url.resolve(this.pickUrl(), options.path); + const url = this.resolveUrl(options.path); const description = options.description || `${options.method} ${url}`; let attempt = 0; const maxAttempts = options.retries ?? DEFAULT_MAX_ATTEMPTS; @@ -107,6 +113,9 @@ export class KbnClientRequester { 'kbn-xsrf': 'kbn-client', }, httpsAgent: this.httpsAgent, + responseType: options.responseType, + // work around https://github.com/axios/axios/issues/2791 + transformResponse: options.responseType === 'text' ? [(x) => x] : undefined, paramsSerializer: (params) => Qs.stringify(params), }); From 15e8ca11618e4af082120a498f4bb8ac4c3bc267 Mon Sep 17 00:00:00 2001 From: Michael Dokolin Date: Fri, 16 Apr 2021 10:09:34 +0200 Subject: [PATCH 09/25] [Reporting] Remove legacy elasticsearch client usage from the reporting plugin (#97184) --- x-pack/plugins/reporting/common/types.ts | 4 +- x-pack/plugins/reporting/server/core.ts | 7 - .../export_types/csv/execute_job.test.ts | 541 ++++++++++-------- .../server/export_types/csv/execute_job.ts | 9 +- .../csv/generate_csv/hit_iterator.test.ts | 59 +- .../csv/generate_csv/hit_iterator.ts | 39 +- .../export_types/csv/generate_csv/index.ts | 8 +- .../png/execute_job/index.test.ts | 10 - .../printable_pdf/execute_job/index.test.ts | 10 - .../reporting/server/lib/store/report.ts | 4 +- .../reporting/server/lib/store/store.test.ts | 79 ++- .../reporting/server/lib/store/store.ts | 186 +++--- x-pack/plugins/reporting/server/plugin.ts | 3 +- .../server/routes/diagnostic/config.test.ts | 23 +- .../server/routes/diagnostic/config.ts | 10 +- .../server/routes/generation.test.ts | 17 +- .../reporting/server/routes/jobs.test.ts | 118 ++-- .../reporting/server/routes/lib/jobs_query.ts | 153 +++-- .../create_mock_reportingplugin.ts | 3 +- 19 files changed, 638 insertions(+), 645 deletions(-) diff --git a/x-pack/plugins/reporting/common/types.ts b/x-pack/plugins/reporting/common/types.ts index 5e20381e35898..2148cf983d889 100644 --- a/x-pack/plugins/reporting/common/types.ts +++ b/x-pack/plugins/reporting/common/types.ts @@ -40,8 +40,8 @@ export interface LayoutParams { export interface ReportDocumentHead { _id: string; _index: string; - _seq_no: unknown; - _primary_term: unknown; + _seq_no: number; + _primary_term: number; } export interface TaskRunResult { diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index b0f0a8c8c7ece..03c76941a6e99 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -10,7 +10,6 @@ import * as Rx from 'rxjs'; import { first, map, take } from 'rxjs/operators'; import { BasePath, - ElasticsearchServiceSetup, IClusterClient, KibanaRequest, PluginInitializerContext, @@ -38,7 +37,6 @@ export interface ReportingInternalSetup { basePath: Pick; router: ReportingPluginRouter; features: FeaturesPluginSetup; - elasticsearch: ElasticsearchServiceSetup; licensing: LicensingPluginSetup; security?: SecurityPluginSetup; spaces?: SpacesPluginSetup; @@ -212,11 +210,6 @@ export class ReportingCore { return this.pluginSetupDeps; } - // NOTE: Uses the Legacy API - public getElasticsearchService() { - return this.getPluginSetupDeps().elasticsearch; - } - private async getSavedObjectsClient(request: KibanaRequest) { const { savedObjects } = await this.getPluginStartDeps(); return savedObjects.getScopedClient(request) as SavedObjectsClientContract; diff --git a/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts b/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts index c0235ee56e00f..f63c07e51dd03 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/execute_job.test.ts @@ -5,8 +5,9 @@ * 2.0. */ +import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import nodeCrypto from '@elastic/node-crypto'; -import { ElasticsearchServiceSetup, IUiSettingsClient } from 'kibana/server'; +import { ElasticsearchClient, IUiSettingsClient } from 'kibana/server'; import moment from 'moment'; // @ts-ignore import Puid from 'puid'; @@ -50,20 +51,12 @@ describe('CSV Execute Job', function () { let defaultElasticsearchResponse: any; let encryptedHeaders: any; - let clusterStub: any; let configGetStub: any; + let mockEsClient: DeeplyMockedKeys; let mockReportingConfig: ReportingConfig; let mockReportingCore: ReportingCore; - let callAsCurrentUserStub: any; let cancellationToken: any; - const mockElasticsearch = { - legacy: { - client: { - asScoped: () => clusterStub, - }, - }, - }; const mockUiSettingsClient = { get: sinon.stub(), }; @@ -85,10 +78,10 @@ describe('CSV Execute Job', function () { mockReportingCore = await createMockReportingCore(mockReportingConfig); mockReportingCore.getUiSettingsServiceFactory = () => Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); - mockReportingCore.getElasticsearchService = () => - mockElasticsearch as ElasticsearchServiceSetup; mockReportingCore.setConfig(mockReportingConfig); + mockEsClient = (await mockReportingCore.getEsClient()).asScoped({} as any) + .asCurrentUser as typeof mockEsClient; cancellationToken = new CancellationToken(); defaultElasticsearchResponse = { @@ -97,14 +90,9 @@ describe('CSV Execute Job', function () { }, _scroll_id: 'defaultScrollId', }; - clusterStub = { - callAsCurrentUser() {}, - }; - - callAsCurrentUserStub = sinon - .stub(clusterStub, 'callAsCurrentUser') - .resolves(defaultElasticsearchResponse); + mockEsClient.search.mockResolvedValue({ body: defaultElasticsearchResponse } as any); + mockEsClient.scroll.mockResolvedValue({ body: defaultElasticsearchResponse } as any); mockUiSettingsClient.get.withArgs(CSV_SEPARATOR_SETTING).returns(','); mockUiSettingsClient.get.withArgs(CSV_QUOTE_VALUES_SETTING).returns(true); @@ -127,7 +115,7 @@ describe('CSV Execute Job', function () { }); describe('basic Elasticsearch call behavior', function () { - it('should decrypt encrypted headers and pass to callAsCurrentUser', async function () { + it('should decrypt encrypted headers and pass to the elasticsearch client', async function () { const runTask = runTaskFnFactory(mockReportingCore, mockLogger); await runTask( 'job456', @@ -138,8 +126,7 @@ describe('CSV Execute Job', function () { }), cancellationToken ); - expect(callAsCurrentUserStub.called).toBe(true); - expect(callAsCurrentUserStub.firstCall.args[0]).toEqual('search'); + expect(mockEsClient.search).toHaveBeenCalled(); }); it('should pass the index and body to execute the initial search', async function () { @@ -160,21 +147,22 @@ describe('CSV Execute Job', function () { await runTask('job777', job, cancellationToken); - const searchCall = callAsCurrentUserStub.firstCall; - expect(searchCall.args[0]).toBe('search'); - expect(searchCall.args[1].index).toBe(index); - expect(searchCall.args[1].body).toBe(body); + expect(mockEsClient.search).toHaveBeenCalledWith(expect.objectContaining({ body, index })); }); it('should pass the scrollId from the initial search to the subsequent scroll', async function () { const scrollId = getRandomScrollId(); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: scrollId, }, - _scroll_id: scrollId, - }); - callAsCurrentUserStub.onSecondCall().resolves(defaultElasticsearchResponse); + } as any); + mockEsClient.scroll.mockResolvedValue({ body: defaultElasticsearchResponse } as any); + const runTask = runTaskFnFactory(mockReportingCore, mockLogger); await runTask( 'job456', @@ -186,10 +174,9 @@ describe('CSV Execute Job', function () { cancellationToken ); - const scrollCall = callAsCurrentUserStub.secondCall; - - expect(scrollCall.args[0]).toBe('scroll'); - expect(scrollCall.args[1].scrollId).toBe(scrollId); + expect(mockEsClient.scroll).toHaveBeenCalledWith( + expect.objectContaining({ scroll_id: scrollId }) + ); }); it('should not execute scroll if there are no hits from the search', async function () { @@ -204,28 +191,27 @@ describe('CSV Execute Job', function () { cancellationToken ); - expect(callAsCurrentUserStub.callCount).toBe(2); - - const searchCall = callAsCurrentUserStub.firstCall; - expect(searchCall.args[0]).toBe('search'); - - const clearScrollCall = callAsCurrentUserStub.secondCall; - expect(clearScrollCall.args[0]).toBe('clearScroll'); + expect(mockEsClient.search).toHaveBeenCalled(); + expect(mockEsClient.clearScroll).toHaveBeenCalled(); }); it('should stop executing scroll if there are no hits', async function () { - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); - callAsCurrentUserStub.onSecondCall().resolves({ - hits: { - hits: [], + } as any); + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + hits: { + hits: [], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); await runTask( @@ -238,33 +224,30 @@ describe('CSV Execute Job', function () { cancellationToken ); - expect(callAsCurrentUserStub.callCount).toBe(3); - - const searchCall = callAsCurrentUserStub.firstCall; - expect(searchCall.args[0]).toBe('search'); - - const scrollCall = callAsCurrentUserStub.secondCall; - expect(scrollCall.args[0]).toBe('scroll'); - - const clearScroll = callAsCurrentUserStub.thirdCall; - expect(clearScroll.args[0]).toBe('clearScroll'); + expect(mockEsClient.search).toHaveBeenCalled(); + expect(mockEsClient.scroll).toHaveBeenCalled(); + expect(mockEsClient.clearScroll).toHaveBeenCalled(); }); it('should call clearScroll with scrollId when there are no more hits', async function () { const lastScrollId = getRandomScrollId(); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); - callAsCurrentUserStub.onSecondCall().resolves({ - hits: { - hits: [], + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + hits: { + hits: [], + }, + _scroll_id: lastScrollId, }, - _scroll_id: lastScrollId, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); await runTask( @@ -277,26 +260,28 @@ describe('CSV Execute Job', function () { cancellationToken ); - const lastCall = callAsCurrentUserStub.getCall(callAsCurrentUserStub.callCount - 1); - expect(lastCall.args[0]).toBe('clearScroll'); - expect(lastCall.args[1].scrollId).toEqual([lastScrollId]); + expect(mockEsClient.clearScroll).toHaveBeenCalledWith( + expect.objectContaining({ scroll_id: lastScrollId }) + ); }); it('calls clearScroll when there is an error iterating the hits', async function () { const lastScrollId = getRandomScrollId(); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [ - { - _source: { - one: 'foo', - two: 'bar', + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [ + { + _source: { + one: 'foo', + two: 'bar', + }, }, - }, - ], + ], + }, + _scroll_id: lastScrollId, }, - _scroll_id: lastScrollId, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -309,21 +294,23 @@ describe('CSV Execute Job', function () { `[TypeError: Cannot read property 'indexOf' of undefined]` ); - const lastCall = callAsCurrentUserStub.getCall(callAsCurrentUserStub.callCount - 1); - expect(lastCall.args[0]).toBe('clearScroll'); - expect(lastCall.args[1].scrollId).toEqual([lastScrollId]); + expect(mockEsClient.clearScroll).toHaveBeenCalledWith( + expect.objectContaining({ scroll_id: lastScrollId }) + ); }); }); describe('Warning when cells have formulas', () => { it('returns `csv_contains_formulas` when cells contain formulas', async function () { configGetStub.withArgs('csv', 'checkForFormulas').returns(true); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: '=SUM(A1:A2)', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: '=SUM(A1:A2)', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -343,12 +330,14 @@ describe('CSV Execute Job', function () { it('returns warnings when headings contain formulas', async function () { configGetStub.withArgs('csv', 'checkForFormulas').returns(true); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { '=SUM(A1:A2)': 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { '=SUM(A1:A2)': 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -369,12 +358,14 @@ describe('CSV Execute Job', function () { it('returns no warnings when cells have no formulas', async function () { configGetStub.withArgs('csv', 'checkForFormulas').returns(true); configGetStub.withArgs('csv', 'escapeFormulaValues').returns(false); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -395,12 +386,14 @@ describe('CSV Execute Job', function () { it('returns no warnings when cells have formulas but are escaped', async function () { configGetStub.withArgs('csv', 'checkForFormulas').returns(true); configGetStub.withArgs('csv', 'escapeFormulaValues').returns(true); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { '=SUM(A1:A2)': 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { '=SUM(A1:A2)': 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -421,12 +414,14 @@ describe('CSV Execute Job', function () { it('returns no warnings when configured not to', async () => { configGetStub.withArgs('csv', 'checkForFormulas').returns(false); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: '=SUM(A1:A2)', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: '=SUM(A1:A2)', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -448,12 +443,14 @@ describe('CSV Execute Job', function () { describe('Byte order mark encoding', () => { it('encodes CSVs with BOM', async () => { configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(true); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: 'one', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'one', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -469,12 +466,14 @@ describe('CSV Execute Job', function () { it('encodes CSVs without BOM', async () => { configGetStub.withArgs('csv', 'useByteOrderMarkEncoding').returns(false); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: 'one', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'one', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -492,12 +491,14 @@ describe('CSV Execute Job', function () { describe('Escaping cells with formulas', () => { it('escapes values with formulas', async () => { configGetStub.withArgs('csv', 'escapeFormulaValues').returns(true); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: `=cmd|' /C calc'!A0`, two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: `=cmd|' /C calc'!A0`, two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -513,12 +514,14 @@ describe('CSV Execute Job', function () { it('does not escapes values with formulas', async () => { configGetStub.withArgs('csv', 'escapeFormulaValues').returns(false); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: `=cmd|' /C calc'!A0`, two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: `=cmd|' /C calc'!A0`, two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -535,7 +538,7 @@ describe('CSV Execute Job', function () { describe('Elasticsearch call errors', function () { it('should reject Promise if search call errors out', async function () { - callAsCurrentUserStub.rejects(new Error()); + mockEsClient.search.mockRejectedValueOnce(new Error()); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -548,13 +551,15 @@ describe('CSV Execute Job', function () { }); it('should reject Promise if scroll call errors out', async function () { - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); - callAsCurrentUserStub.onSecondCall().rejects(new Error()); + } as any); + mockEsClient.scroll.mockRejectedValueOnce(new Error()); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -569,12 +574,14 @@ describe('CSV Execute Job', function () { describe('invalid responses', function () { it('should reject Promise if search returns hits but no _scroll_id', async function () { - callAsCurrentUserStub.resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: undefined, }, - _scroll_id: undefined, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -588,12 +595,14 @@ describe('CSV Execute Job', function () { }); it('should reject Promise if search returns no hits and no _scroll_id', async function () { - callAsCurrentUserStub.resolves({ - hits: { - hits: [], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [], + }, + _scroll_id: undefined, }, - _scroll_id: undefined, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -607,19 +616,23 @@ describe('CSV Execute Job', function () { }); it('should reject Promise if scroll returns hits but no _scroll_id', async function () { - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); - callAsCurrentUserStub.onSecondCall().resolves({ - hits: { - hits: [{}], + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: undefined, }, - _scroll_id: undefined, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -633,19 +646,23 @@ describe('CSV Execute Job', function () { }); it('should reject Promise if scroll returns no hits and no _scroll_id', async function () { - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); - callAsCurrentUserStub.onSecondCall().resolves({ - hits: { - hits: [], + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + hits: { + hits: [], + }, + _scroll_id: undefined, }, - _scroll_id: undefined, - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -663,21 +680,20 @@ describe('CSV Execute Job', function () { const scrollId = getRandomScrollId(); beforeEach(function () { - // We have to "re-stub" the callAsCurrentUser stub here so that we can use the fakeFunction - // that delays the Promise resolution so we have a chance to call cancellationToken.cancel(). - // Otherwise, we get into an endless loop, and don't have a chance to call cancel - callAsCurrentUserStub.restore(); - callAsCurrentUserStub = sinon - .stub(clusterStub, 'callAsCurrentUser') - .callsFake(async function () { - await delay(1); - return { + const searchStub = async () => { + await delay(1); + return { + body: { hits: { hits: [{}], }, _scroll_id: scrollId, - }; - }); + }, + }; + }; + + mockEsClient.search.mockImplementation(searchStub as typeof mockEsClient.search); + mockEsClient.scroll.mockImplementation(searchStub as typeof mockEsClient.scroll); }); it('should stop calling Elasticsearch when cancellationToken.cancel is called', async function () { @@ -693,10 +709,15 @@ describe('CSV Execute Job', function () { ); await delay(250); - const callCount = callAsCurrentUserStub.callCount; + + expect(mockEsClient.search).toHaveBeenCalled(); + expect(mockEsClient.scroll).toHaveBeenCalled(); + expect(mockEsClient.clearScroll).not.toHaveBeenCalled(); + cancellationToken.cancel(); await delay(250); - expect(callAsCurrentUserStub.callCount).toBe(callCount + 1); // last call is to clear the scroll + + expect(mockEsClient.clearScroll).toHaveBeenCalled(); }); it(`shouldn't call clearScroll if it never got a scrollId`, async function () { @@ -712,9 +733,7 @@ describe('CSV Execute Job', function () { ); cancellationToken.cancel(); - for (let i = 0; i < callAsCurrentUserStub.callCount; ++i) { - expect(callAsCurrentUserStub.getCall(i).args[1]).not.toBe('clearScroll'); // dead code? - } + expect(mockEsClient.clearScroll).not.toHaveBeenCalled(); }); it('should call clearScroll if it got a scrollId', async function () { @@ -732,9 +751,11 @@ describe('CSV Execute Job', function () { cancellationToken.cancel(); await delay(100); - const lastCall = callAsCurrentUserStub.getCall(callAsCurrentUserStub.callCount - 1); - expect(lastCall.args[0]).toBe('clearScroll'); - expect(lastCall.args[1].scrollId).toEqual([scrollId]); + expect(mockEsClient.clearScroll).toHaveBeenCalledWith( + expect.objectContaining({ + scroll_id: scrollId, + }) + ); }); }); @@ -788,12 +809,14 @@ describe('CSV Execute Job', function () { it('should write column headers to output, when there are results', async function () { const runTask = runTaskFnFactory(mockReportingCore, mockLogger); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{ one: '1', two: '2' }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ one: '1', two: '2' }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -809,12 +832,14 @@ describe('CSV Execute Job', function () { it('should use comma separated values of non-nested fields from _source', async function () { const runTask = runTaskFnFactory(mockReportingCore, mockLogger); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -831,18 +856,22 @@ describe('CSV Execute Job', function () { it('should concatenate the hits from multiple responses', async function () { const runTask = runTaskFnFactory(mockReportingCore, mockLogger); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); - callAsCurrentUserStub.onSecondCall().resolves({ - hits: { - hits: [{ _source: { one: 'baz', two: 'qux' } }], + } as any); + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'baz', two: 'qux' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -860,12 +889,14 @@ describe('CSV Execute Job', function () { it('should use field formatters to format fields', async function () { const runTask = runTaskFnFactory(mockReportingCore, mockLogger); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const jobParams = getBasePayload({ headers: encryptedHeaders, @@ -962,12 +993,14 @@ describe('CSV Execute Job', function () { beforeEach(async function () { configGetStub.withArgs('csv', 'maxSizeBytes').returns(9); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -1002,12 +1035,14 @@ describe('CSV Execute Job', function () { Promise.resolve((mockUiSettingsClient as unknown) as IUiSettingsClient); configGetStub.withArgs('csv', 'maxSizeBytes').returns(18); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{ _source: { one: 'foo', two: 'bar' } }], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{ _source: { one: 'foo', two: 'bar' } }], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -1039,12 +1074,14 @@ describe('CSV Execute Job', function () { const scrollDuration = 'test'; configGetStub.withArgs('csv', 'scroll').returns({ duration: scrollDuration }); - callAsCurrentUserStub.onFirstCall().returns({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -1056,21 +1093,23 @@ describe('CSV Execute Job', function () { await runTask('job123', jobParams, cancellationToken); - const searchCall = callAsCurrentUserStub.firstCall; - expect(searchCall.args[0]).toBe('search'); - expect(searchCall.args[1].scroll).toBe(scrollDuration); + expect(mockEsClient.search).toHaveBeenCalledWith( + expect.objectContaining({ scroll: scrollDuration }) + ); }); it('passes scroll size to initial search call', async function () { const scrollSize = 100; configGetStub.withArgs('csv', 'scroll').returns({ size: scrollSize }); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -1082,21 +1121,23 @@ describe('CSV Execute Job', function () { await runTask('job123', jobParams, cancellationToken); - const searchCall = callAsCurrentUserStub.firstCall; - expect(searchCall.args[0]).toBe('search'); - expect(searchCall.args[1].size).toBe(scrollSize); + expect(mockEsClient.search).toHaveBeenCalledWith( + expect.objectContaining({ size: scrollSize }) + ); }); it('passes scroll duration to subsequent scroll call', async function () { const scrollDuration = 'test'; configGetStub.withArgs('csv', 'scroll').returns({ duration: scrollDuration }); - callAsCurrentUserStub.onFirstCall().resolves({ - hits: { - hits: [{}], + mockEsClient.search.mockResolvedValueOnce({ + body: { + hits: { + hits: [{}], + }, + _scroll_id: 'scrollId', }, - _scroll_id: 'scrollId', - }); + } as any); const runTask = runTaskFnFactory(mockReportingCore, mockLogger); const jobParams = getBasePayload({ @@ -1108,9 +1149,9 @@ describe('CSV Execute Job', function () { await runTask('job123', jobParams, cancellationToken); - const scrollCall = callAsCurrentUserStub.secondCall; - expect(scrollCall.args[0]).toBe('scroll'); - expect(scrollCall.args[1].scroll).toBe(scrollDuration); + expect(mockEsClient.scroll).toHaveBeenCalledWith( + expect.objectContaining({ scroll: scrollDuration }) + ); }); }); }); diff --git a/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts index 0e13a91649406..57559d136ff3e 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts @@ -17,7 +17,7 @@ export const runTaskFnFactory: RunTaskFnFactory< const config = reporting.getConfig(); return async function runTask(jobId, job, cancellationToken) { - const elasticsearch = reporting.getElasticsearchService(); + const elasticsearch = await reporting.getEsClient(); const logger = parentLogger.clone([jobId]); const generateCsv = createGenerateCsv(logger); @@ -25,16 +25,13 @@ export const runTaskFnFactory: RunTaskFnFactory< const headers = await decryptJobHeaders(encryptionKey, job.headers, logger); const fakeRequest = reporting.getFakeRequest({ headers }, job.spaceId, logger); const uiSettingsClient = await reporting.getUiSettingsClient(fakeRequest, logger); - - const { callAsCurrentUser } = elasticsearch.legacy.client.asScoped(fakeRequest); - const callEndpoint = (endpoint: string, clientParams = {}, options = {}) => - callAsCurrentUser(endpoint, clientParams, options); + const { asCurrentUser: elasticsearchClient } = elasticsearch.asScoped(fakeRequest); const { content, maxSizeReached, size, csvContainsFormulas, warnings } = await generateCsv( job, config, uiSettingsClient, - callEndpoint, + elasticsearchClient, cancellationToken ); diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.test.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.test.ts index 4baa81e8be6c9..6b1b7fc98a4b8 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.test.ts @@ -7,16 +7,18 @@ import expect from '@kbn/expect'; import sinon from 'sinon'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { CancellationToken } from '../../../../common'; import { createMockLevelLogger } from '../../../test_helpers/create_mock_levellogger'; import { ScrollConfig } from '../../../types'; import { createHitIterator } from './hit_iterator'; +const { asInternalUser: mockEsClient } = elasticsearchServiceMock.createClusterClient(); const mockLogger = createMockLevelLogger(); const debugLogStub = sinon.stub(mockLogger, 'debug'); const warnLogStub = sinon.stub(mockLogger, 'warn'); const errorLogStub = sinon.stub(mockLogger, 'error'); -const mockCallEndpoint = sinon.stub(); + const mockSearchRequest = {}; const mockConfig: ScrollConfig = { duration: '2s', size: 123 }; let realCancellationToken = new CancellationToken(); @@ -27,10 +29,30 @@ describe('hitIterator', function () { debugLogStub.resetHistory(); warnLogStub.resetHistory(); errorLogStub.resetHistory(); - mockCallEndpoint.resetHistory(); - mockCallEndpoint.resetBehavior(); - mockCallEndpoint.resolves({ _scroll_id: '123blah', hits: { hits: ['you found me'] } }); - mockCallEndpoint.onCall(11).resolves({ _scroll_id: '123blah', hits: {} }); + + mockEsClient.search.mockClear(); + mockEsClient.search.mockResolvedValue({ + body: { + _scroll_id: '123blah', + hits: { hits: ['you found me'] }, + }, + } as any); + + mockEsClient.scroll.mockClear(); + for (let i = 0; i < 10; i++) { + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + _scroll_id: '123blah', + hits: { hits: ['you found me'] }, + }, + } as any); + } + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + _scroll_id: '123blah', + hits: {}, + }, + } as any); isCancelledStub = sinon.stub(realCancellationToken, 'isCancelled'); isCancelledStub.returns(false); @@ -45,7 +67,7 @@ describe('hitIterator', function () { const hitIterator = createHitIterator(mockLogger); const iterator = hitIterator( mockConfig, - mockCallEndpoint, + mockEsClient, mockSearchRequest, realCancellationToken ); @@ -58,7 +80,7 @@ describe('hitIterator', function () { expect(hit).to.be('you found me'); } - expect(mockCallEndpoint.callCount).to.be(13); + expect(mockEsClient.scroll.mock.calls.length).to.be(11); expect(debugLogStub.callCount).to.be(13); expect(warnLogStub.callCount).to.be(0); expect(errorLogStub.callCount).to.be(0); @@ -73,7 +95,7 @@ describe('hitIterator', function () { const hitIterator = createHitIterator(mockLogger); const iterator = hitIterator( mockConfig, - mockCallEndpoint, + mockEsClient, mockSearchRequest, realCancellationToken ); @@ -86,7 +108,7 @@ describe('hitIterator', function () { expect(hit).to.be('you found me'); } - expect(mockCallEndpoint.callCount).to.be(3); + expect(mockEsClient.scroll.mock.calls.length).to.be(1); expect(debugLogStub.callCount).to.be(3); expect(warnLogStub.callCount).to.be(1); expect(errorLogStub.callCount).to.be(0); @@ -98,13 +120,20 @@ describe('hitIterator', function () { it('handles time out', async () => { // Setup - mockCallEndpoint.onCall(2).resolves({ status: 404 }); + mockEsClient.scroll.mockReset(); + mockEsClient.scroll.mockResolvedValueOnce({ + body: { + _scroll_id: '123blah', + hits: { hits: ['you found me'] }, + }, + } as any); + mockEsClient.scroll.mockResolvedValueOnce({ body: { status: 404 } } as any); // Begin const hitIterator = createHitIterator(mockLogger); const iterator = hitIterator( mockConfig, - mockCallEndpoint, + mockEsClient, mockSearchRequest, realCancellationToken ); @@ -125,7 +154,7 @@ describe('hitIterator', function () { errorThrown = true; } - expect(mockCallEndpoint.callCount).to.be(4); + expect(mockEsClient.scroll.mock.calls.length).to.be(2); expect(debugLogStub.callCount).to.be(4); expect(warnLogStub.callCount).to.be(0); expect(errorLogStub.callCount).to.be(1); @@ -134,13 +163,13 @@ describe('hitIterator', function () { it('handles scroll id could not be cleared', async () => { // Setup - mockCallEndpoint.withArgs('clearScroll').rejects({ status: 404 }); + mockEsClient.clearScroll.mockRejectedValueOnce({ status: 404 }); // Begin const hitIterator = createHitIterator(mockLogger); const iterator = hitIterator( mockConfig, - mockCallEndpoint, + mockEsClient, mockSearchRequest, realCancellationToken ); @@ -153,7 +182,7 @@ describe('hitIterator', function () { expect(hit).to.be('you found me'); } - expect(mockCallEndpoint.callCount).to.be(13); + expect(mockEsClient.scroll.mock.calls.length).to.be(11); expect(warnLogStub.callCount).to.be(1); expect(errorLogStub.callCount).to.be(1); }); diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts index b00622399d691..72935e64dd6b5 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/hit_iterator.ts @@ -5,54 +5,55 @@ * 2.0. */ +import { UnwrapPromise } from '@kbn/utility-types'; import { i18n } from '@kbn/i18n'; -import { SearchParams, SearchResponse } from 'elasticsearch'; +import { ElasticsearchClient } from 'src/core/server'; import { CancellationToken } from '../../../../common'; import { LevelLogger } from '../../../lib'; import { ScrollConfig } from '../../../types'; -export type EndpointCaller = (method: string, params: object) => Promise>; +type SearchResponse = UnwrapPromise>; +type SearchRequest = Required>[0]; -function parseResponse(request: SearchResponse) { - const response = request; - if (!response || !response._scroll_id) { +function parseResponse(response: SearchResponse) { + if (!response?.body._scroll_id) { throw new Error( i18n.translate('xpack.reporting.exportTypes.csv.hitIterator.expectedScrollIdErrorMessage', { defaultMessage: 'Expected {scrollId} in the following Elasticsearch response: {response}', - values: { response: JSON.stringify(response), scrollId: '_scroll_id' }, + values: { response: JSON.stringify(response?.body), scrollId: '_scroll_id' }, }) ); } - if (!response.hits) { + if (!response?.body.hits) { throw new Error( i18n.translate('xpack.reporting.exportTypes.csv.hitIterator.expectedHitsErrorMessage', { defaultMessage: 'Expected {hits} in the following Elasticsearch response: {response}', - values: { response: JSON.stringify(response), hits: 'hits' }, + values: { response: JSON.stringify(response?.body), hits: 'hits' }, }) ); } return { - scrollId: response._scroll_id, - hits: response.hits.hits, + scrollId: response.body._scroll_id, + hits: response.body.hits.hits, }; } export function createHitIterator(logger: LevelLogger) { return async function* hitIterator( scrollSettings: ScrollConfig, - callEndpoint: EndpointCaller, - searchRequest: SearchParams, + elasticsearchClient: ElasticsearchClient, + searchRequest: SearchRequest, cancellationToken: CancellationToken ) { logger.debug('executing search request'); - async function search(index: string | boolean | string[] | undefined, body: object) { + async function search(index: SearchRequest['index'], body: SearchRequest['body']) { return parseResponse( - await callEndpoint('search', { - ignore_unavailable: true, // ignores if the index pattern contains any aliases that point to closed indices + await elasticsearchClient.search({ index, body, + ignore_unavailable: true, // ignores if the index pattern contains any aliases that point to closed indices scroll: scrollSettings.duration, size: scrollSettings.size, }) @@ -62,8 +63,8 @@ export function createHitIterator(logger: LevelLogger) { async function scroll(scrollId: string | undefined) { logger.debug('executing scroll request'); return parseResponse( - await callEndpoint('scroll', { - scrollId, + await elasticsearchClient.scroll({ + scroll_id: scrollId, scroll: scrollSettings.duration, }) ); @@ -72,8 +73,8 @@ export function createHitIterator(logger: LevelLogger) { async function clearScroll(scrollId: string | undefined) { logger.debug('executing clearScroll request'); try { - await callEndpoint('clearScroll', { - scrollId: [scrollId], + await elasticsearchClient.clearScroll({ + scroll_id: scrollId, }); } catch (err) { // Do not throw the error, as the job can still be completed successfully diff --git a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts index 629a81df350be..e5ed04f4cab66 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { IUiSettingsClient } from 'src/core/server'; +import { ElasticsearchClient, IUiSettingsClient } from 'src/core/server'; import { ReportingConfig } from '../../../'; import { CancellationToken } from '../../../../../../plugins/reporting/common'; import { CSV_BOM_CHARS } from '../../../../common/constants'; @@ -24,7 +24,7 @@ import { fieldFormatMapFactory } from './field_format_map'; import { createFlattenHit } from './flatten_hit'; import { createFormatCsvValues } from './format_csv_values'; import { getUiSettings } from './get_ui_settings'; -import { createHitIterator, EndpointCaller } from './hit_iterator'; +import { createHitIterator } from './hit_iterator'; interface SearchRequest { index: string; @@ -56,7 +56,7 @@ export function createGenerateCsv(logger: LevelLogger) { job: GenerateCsvParams, config: ReportingConfig, uiSettingsClient: IUiSettingsClient, - callEndpoint: EndpointCaller, + elasticsearchClient: ElasticsearchClient, cancellationToken: CancellationToken ): Promise { const settings = await getUiSettings(job.browserTimezone, uiSettingsClient, config, logger); @@ -79,7 +79,7 @@ export function createGenerateCsv(logger: LevelLogger) { const iterator = hitIterator( settings.scroll, - callEndpoint, + elasticsearchClient, job.searchRequest, cancellationToken ); diff --git a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts index 3a5298981738d..34fe5360522b1 100644 --- a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts +++ b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.test.ts @@ -59,16 +59,6 @@ beforeEach(async () => { mockReporting = await createMockReportingCore(mockReportingConfig); - const mockElasticsearch = { - legacy: { - client: { - asScoped: () => ({ callAsCurrentUser: jest.fn() }), - }, - }, - }; - const mockGetElasticsearch = jest.fn(); - mockGetElasticsearch.mockImplementation(() => Promise.resolve(mockElasticsearch)); - mockReporting.getElasticsearchService = mockGetElasticsearch; // @ts-ignore over-riding config method mockReporting.config = mockReportingConfig; diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.test.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.test.ts index 0c6a55fb895b5..61eab18987f7c 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.test.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.test.ts @@ -57,16 +57,6 @@ beforeEach(async () => { mockReporting = await createMockReportingCore(mockReportingConfig); - const mockElasticsearch = { - legacy: { - client: { - asScoped: () => ({ callAsCurrentUser: jest.fn() }), - }, - }, - }; - const mockGetElasticsearch = jest.fn(); - mockGetElasticsearch.mockImplementation(() => Promise.resolve(mockElasticsearch)); - mockReporting.getElasticsearchService = mockGetElasticsearch; // @ts-ignore over-riding config mockReporting.config = mockReportingConfig; diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts index 817028cab1a39..9b98650e1d984 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -25,8 +25,8 @@ const puid = new Puid(); export class Report implements Partial { public _index?: string; public _id: string; - public _primary_term?: unknown; // set by ES - public _seq_no: unknown; // set by ES + public _primary_term?: number; // set by ES + public _seq_no?: number; // set by ES public readonly kibana_name: ReportSource['kibana_name']; public readonly kibana_id: ReportSource['kibana_id']; diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 01d91f8bc2ac2..2af0fe7830eea 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -5,8 +5,8 @@ * 2.0. */ -import sinon from 'sinon'; -import { ElasticsearchServiceSetup } from 'src/core/server'; +import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; +import { ElasticsearchClient } from 'src/core/server'; import { ReportingConfig, ReportingCore } from '../../'; import { createMockConfig, @@ -21,9 +21,7 @@ describe('ReportingStore', () => { const mockLogger = createMockLevelLogger(); let mockConfig: ReportingConfig; let mockCore: ReportingCore; - - const callClusterStub = sinon.stub(); - const mockElasticsearch = { legacy: { client: { callAsInternalUser: callClusterStub } } }; + let mockEsClient: DeeplyMockedKeys; beforeEach(async () => { const reportingConfig = { @@ -33,17 +31,14 @@ describe('ReportingStore', () => { const mockSchema = createMockConfigSchema(reportingConfig); mockConfig = createMockConfig(mockSchema); mockCore = await createMockReportingCore(mockConfig); + mockEsClient = (await mockCore.getEsClient()).asInternalUser as typeof mockEsClient; - callClusterStub.reset(); - callClusterStub.withArgs('indices.exists').resolves({}); - callClusterStub.withArgs('indices.create').resolves({}); - callClusterStub.withArgs('index').resolves({ _id: 'stub-id', _index: 'stub-index' }); - callClusterStub.withArgs('indices.refresh').resolves({}); - callClusterStub.withArgs('update').resolves({}); - callClusterStub.withArgs('get').resolves({}); - - mockCore.getElasticsearchService = () => - (mockElasticsearch as unknown) as ElasticsearchServiceSetup; + mockEsClient.indices.create.mockResolvedValue({} as any); + mockEsClient.indices.exists.mockResolvedValue({} as any); + mockEsClient.indices.refresh.mockResolvedValue({} as any); + mockEsClient.get.mockResolvedValue({} as any); + mockEsClient.index.mockResolvedValue({ body: { _id: 'stub-id', _index: 'stub-index' } } as any); + mockEsClient.update.mockResolvedValue({} as any); }); describe('addReport', () => { @@ -88,14 +83,14 @@ describe('ReportingStore', () => { meta: {}, } as any); expect(store.addReport(mockReport)).rejects.toMatchInlineSnapshot( - `[TypeError: this.client.callAsInternalUser is not a function]` + `[Error: Report object from ES has missing fields!]` ); }); it('handles error creating the index', async () => { // setup - callClusterStub.withArgs('indices.exists').resolves(false); - callClusterStub.withArgs('indices.create').rejects(new Error('horrible error')); + mockEsClient.indices.exists.mockResolvedValue({ body: false } as any); + mockEsClient.indices.create.mockRejectedValue(new Error('horrible error')); const store = new ReportingStore(mockCore, mockLogger); const mockReport = new Report({ @@ -117,8 +112,8 @@ describe('ReportingStore', () => { */ it('ignores index creation error if the index already exists and continues adding the report', async () => { // setup - callClusterStub.withArgs('indices.exists').resolves(false); - callClusterStub.withArgs('indices.create').rejects(new Error('devastating error')); + mockEsClient.indices.exists.mockResolvedValue({ body: false } as any); + mockEsClient.indices.create.mockRejectedValue(new Error('devastating error')); const store = new ReportingStore(mockCore, mockLogger); const mockReport = new Report({ @@ -134,10 +129,9 @@ describe('ReportingStore', () => { it('skips creating the index if already exists', async () => { // setup - callClusterStub.withArgs('indices.exists').resolves(false); - callClusterStub - .withArgs('indices.create') - .rejects(new Error('resource_already_exists_exception')); // will be triggered but ignored + mockEsClient.indices.exists.mockResolvedValue({ body: false } as any); + // will be triggered but ignored + mockEsClient.indices.create.mockRejectedValue(new Error('resource_already_exists_exception')); const store = new ReportingStore(mockCore, mockLogger); const mockReport = new Report({ @@ -159,10 +153,9 @@ describe('ReportingStore', () => { it('allows username string to be `false`', async () => { // setup - callClusterStub.withArgs('indices.exists').resolves(false); - callClusterStub - .withArgs('indices.create') - .rejects(new Error('resource_already_exists_exception')); // will be triggered but ignored + mockEsClient.indices.exists.mockResolvedValue({ body: false } as any); + // will be triggered but ignored + mockEsClient.indices.create.mockRejectedValue(new Error('resource_already_exists_exception')); const store = new ReportingStore(mockCore, mockLogger); const mockReport = new Report({ @@ -192,8 +185,8 @@ describe('ReportingStore', () => { const mockReport: ReportDocument = { _id: '1234-foo-78', _index: '.reporting-test-17409', - _primary_term: 'primary_term string', - _seq_no: 'seq_no string', + _primary_term: 1234, + _seq_no: 5678, _source: { kibana_name: 'test', kibana_id: 'test123', @@ -210,7 +203,7 @@ describe('ReportingStore', () => { output: null, }, }; - callClusterStub.withArgs('get').resolves(mockReport); + mockEsClient.get.mockResolvedValue({ body: mockReport } as any); const store = new ReportingStore(mockCore, mockLogger); const report = new Report({ ...mockReport, @@ -221,8 +214,8 @@ describe('ReportingStore', () => { Report { "_id": "1234-foo-78", "_index": ".reporting-test-17409", - "_primary_term": "primary_term string", - "_seq_no": "seq_no string", + "_primary_term": 1234, + "_seq_no": 5678, "attempts": 0, "browser_type": "browser type string", "completed_at": undefined, @@ -267,10 +260,9 @@ describe('ReportingStore', () => { await store.setReportClaimed(report, { testDoc: 'test' } as any); - const updateCall = callClusterStub.getCalls().find((call) => call.args[0] === 'update'); - expect(updateCall && updateCall.args).toMatchInlineSnapshot(` + const [updateCall] = mockEsClient.update.mock.calls; + expect(updateCall).toMatchInlineSnapshot(` Array [ - "update", Object { "body": Object { "doc": Object { @@ -308,10 +300,9 @@ describe('ReportingStore', () => { await store.setReportFailed(report, { errors: 'yes' } as any); - const updateCall = callClusterStub.getCalls().find((call) => call.args[0] === 'update'); - expect(updateCall && updateCall.args).toMatchInlineSnapshot(` + const [updateCall] = mockEsClient.update.mock.calls; + expect(updateCall).toMatchInlineSnapshot(` Array [ - "update", Object { "body": Object { "doc": Object { @@ -349,10 +340,9 @@ describe('ReportingStore', () => { await store.setReportCompleted(report, { certainly_completed: 'yes' } as any); - const updateCall = callClusterStub.getCalls().find((call) => call.args[0] === 'update'); - expect(updateCall && updateCall.args).toMatchInlineSnapshot(` + const [updateCall] = mockEsClient.update.mock.calls; + expect(updateCall).toMatchInlineSnapshot(` Array [ - "update", Object { "body": Object { "doc": Object { @@ -395,10 +385,9 @@ describe('ReportingStore', () => { }, } as any); - const updateCall = callClusterStub.getCalls().find((call) => call.args[0] === 'update'); - expect(updateCall && updateCall.args).toMatchInlineSnapshot(` + const [updateCall] = mockEsClient.update.mock.calls; + expect(updateCall).toMatchInlineSnapshot(` Array [ - "update", Object { "body": Object { "doc": Object { diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index fdac471c26cb0..fc7bd9c23d769 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -5,8 +5,7 @@ * 2.0. */ -import { SearchParams } from 'elasticsearch'; -import { ElasticsearchServiceSetup } from 'src/core/server'; +import { ElasticsearchClient } from 'src/core/server'; import { LevelLogger, statuses } from '../'; import { ReportingCore } from '../../'; import { numberToDuration } from '../../../common/schema_utils'; @@ -14,7 +13,7 @@ import { JobStatus } from '../../../common/types'; import { ReportTaskParams } from '../tasks'; import { indexTimestamp } from './index_timestamp'; import { mapping } from './mapping'; -import { Report, ReportDocument } from './report'; +import { Report, ReportDocument, ReportSource } from './report'; /* * When searching for long-pending reports, we get a subset of fields @@ -45,59 +44,60 @@ export class ReportingStore { private readonly indexPrefix: string; // config setting of index prefix in system index name private readonly indexInterval: string; // config setting of index prefix: how often to poll for pending work private readonly queueTimeoutMins: number; // config setting of queue timeout, rounded up to nearest minute - private client: ElasticsearchServiceSetup['legacy']['client']; - private logger: LevelLogger; + private client?: ElasticsearchClient; - constructor(reporting: ReportingCore, logger: LevelLogger) { - const config = reporting.getConfig(); - const elasticsearch = reporting.getElasticsearchService(); + constructor(private reportingCore: ReportingCore, private logger: LevelLogger) { + const config = reportingCore.getConfig(); - this.client = elasticsearch.legacy.client; this.indexPrefix = config.get('index'); this.indexInterval = config.get('queue', 'indexInterval'); this.logger = logger.clone(['store']); this.queueTimeoutMins = Math.ceil(numberToDuration(config.get('queue', 'timeout')).asMinutes()); } + private async getClient() { + if (!this.client) { + ({ asInternalUser: this.client } = await this.reportingCore.getEsClient()); + } + + return this.client; + } + private async createIndex(indexName: string) { - return await this.client - .callAsInternalUser('indices.exists', { - index: indexName, - }) - .then((exists) => { - if (exists) { - return exists; - } - - const indexSettings = { - number_of_shards: 1, - auto_expand_replicas: '0-1', - }; - const body = { - settings: indexSettings, - mappings: { - properties: mapping, - }, - }; - - return this.client - .callAsInternalUser('indices.create', { - index: indexName, - body, - }) - .then(() => true) - .catch((err: Error) => { - const isIndexExistsError = err.message.match(/resource_already_exists_exception/); - if (isIndexExistsError) { - // Do not fail a job if the job runner hits the race condition. - this.logger.warn(`Automatic index creation failed: index already exists: ${err}`); - return; - } - - this.logger.error(err); - throw err; - }); - }); + const client = await this.getClient(); + const { body: exists } = await client.indices.exists({ index: indexName }); + + if (exists) { + return exists; + } + + const indexSettings = { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }; + const body = { + settings: indexSettings, + mappings: { + properties: mapping, + }, + }; + + try { + await client.indices.create({ index: indexName, body }); + + return true; + } catch (error) { + const isIndexExistsError = error.message.match(/resource_already_exists_exception/); + if (isIndexExistsError) { + // Do not fail a job if the job runner hits the race condition. + this.logger.warn(`Automatic index creation failed: index already exists: ${error}`); + return; + } + + this.logger.error(error); + + throw error; + } } /* @@ -105,7 +105,7 @@ export class ReportingStore { */ private async indexReport(report: Report) { const doc = { - index: report._index, + index: report._index!, id: report._id, body: { ...report.toEsDocsJSON()._source, @@ -114,14 +114,20 @@ export class ReportingStore { status: statuses.JOB_STATUS_PENDING, }, }; - return await this.client.callAsInternalUser('index', doc); + + const client = await this.getClient(); + const { body } = await client.index(doc); + + return body; } /* * Called from addReport, which handles any errors */ private async refreshIndex(index: string) { - return await this.client.callAsInternalUser('indices.refresh', { index }); + const client = await this.getClient(); + + return client.indices.refresh({ index }); } public async addReport(report: Report): Promise { @@ -156,7 +162,8 @@ export class ReportingStore { } try { - const document = await this.client.callAsInternalUser('get', { + const client = await this.getClient(); + const { body: document } = await client.get({ index: taskJson.index, id: taskJson.id, }); @@ -166,17 +173,17 @@ export class ReportingStore { _index: document._index, _seq_no: document._seq_no, _primary_term: document._primary_term, - jobtype: document._source.jobtype, - attempts: document._source.attempts, - browser_type: document._source.browser_type, - created_at: document._source.created_at, - created_by: document._source.created_by, - max_attempts: document._source.max_attempts, - meta: document._source.meta, - payload: document._source.payload, - process_expiration: document._source.process_expiration, - status: document._source.status, - timeout: document._source.timeout, + jobtype: document._source?.jobtype, + attempts: document._source?.attempts, + browser_type: document._source?.browser_type, + created_at: document._source?.created_at, + created_by: document._source?.created_by, + max_attempts: document._source?.max_attempts, + meta: document._source?.meta, + payload: document._source?.payload, + process_expiration: document._source?.process_expiration, + status: document._source?.status, + timeout: document._source?.timeout, }); } catch (err) { this.logger.error('Error in finding a report! ' + JSON.stringify({ report: taskJson })); @@ -191,14 +198,17 @@ export class ReportingStore { try { checkReportIsEditable(report); - return await this.client.callAsInternalUser('update', { + const client = await this.getClient(); + const { body } = await client.update({ id: report._id, - index: report._index, + index: report._index!, if_seq_no: report._seq_no, if_primary_term: report._primary_term, refresh: true, body: { doc }, }); + + return (body as unknown) as ReportDocument; } catch (err) { this.logger.error('Error in setting report pending status!'); this.logger.error(err); @@ -215,14 +225,17 @@ export class ReportingStore { try { checkReportIsEditable(report); - return await this.client.callAsInternalUser('update', { + const client = await this.getClient(); + const { body } = await client.update({ id: report._id, - index: report._index, + index: report._index!, if_seq_no: report._seq_no, if_primary_term: report._primary_term, refresh: true, body: { doc }, }); + + return (body as unknown) as ReportDocument; } catch (err) { this.logger.error('Error in setting report processing status!'); this.logger.error(err); @@ -239,14 +252,17 @@ export class ReportingStore { try { checkReportIsEditable(report); - return await this.client.callAsInternalUser('update', { + const client = await this.getClient(); + const { body } = await client.update({ id: report._id, - index: report._index, + index: report._index!, if_seq_no: report._seq_no, if_primary_term: report._primary_term, refresh: true, body: { doc }, }); + + return (body as unknown) as ReportDocument; } catch (err) { this.logger.error('Error in setting report failed status!'); this.logger.error(err); @@ -267,14 +283,17 @@ export class ReportingStore { }; checkReportIsEditable(report); - return await this.client.callAsInternalUser('update', { + const client = await this.getClient(); + const { body } = await client.update({ id: report._id, - index: report._index, + index: report._index!, if_seq_no: report._seq_no, if_primary_term: report._primary_term, refresh: true, body: { doc }, }); + + return (body as unknown) as ReportDocument; } catch (err) { this.logger.error('Error in setting report complete status!'); this.logger.error(err); @@ -286,16 +305,17 @@ export class ReportingStore { try { checkReportIsEditable(report); - const updateParams = { + const client = await this.getClient(); + const { body } = await client.update({ id: report._id, - index: report._index, + index: report._index!, if_seq_no: report._seq_no, if_primary_term: report._primary_term, refresh: true, body: { doc: { process_expiration: null } }, - }; + }); - return await this.client.callAsInternalUser('update', updateParams); + return (body as unknown) as ReportDocument; } catch (err) { this.logger.error('Error in clearing expiration!'); this.logger.error(err); @@ -312,12 +332,11 @@ export class ReportingStore { * Pending reports are not included in this search: they may be scheduled in TM just not run yet. * TODO Should we get a list of the reports that are pending and scheduled in TM so we can exclude them from this query? */ - public async findZombieReportDocuments( - logger = this.logger - ): Promise { - const searchParams: SearchParams = { + public async findZombieReportDocuments(): Promise { + const client = await this.getClient(); + const { body } = await client.search({ index: this.indexPrefix + '-*', - filterPath: 'hits.hits', + filter_path: 'hits.hits', body: { sort: { created_at: { order: 'desc' } }, query: { @@ -335,13 +354,8 @@ export class ReportingStore { }, }, }, - }; - - const result = await this.client.callAsInternalUser( - 'search', - searchParams - ); + }); - return result.hits?.hits; + return body.hits?.hits as ReportRecordTimeout[]; } } diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index 3dc7e7ef3df92..75411b30ec0bd 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -47,7 +47,7 @@ export class ReportingPlugin registerUiSettings(core); - const { elasticsearch, http } = core; + const { http } = core; const { features, licensing, security, spaces, taskManager } = plugins; const { initializerContext: initContext, reportingCore } = this; @@ -56,7 +56,6 @@ export class ReportingPlugin reportingCore.pluginSetup({ features, - elasticsearch, licensing, basePath, router, diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts index f35d8f5910da0..952a33ff64190 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts @@ -6,6 +6,8 @@ */ import { UnwrapPromise } from '@kbn/utility-types'; +import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; +import { ElasticsearchClient } from 'kibana/server'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '../..'; @@ -26,6 +28,7 @@ describe('POST /diagnose/config', () => { let core: ReportingCore; let mockSetupDeps: any; let config: any; + let mockEsClient: DeeplyMockedKeys; const mockLogger = createMockLevelLogger(); @@ -38,9 +41,6 @@ describe('POST /diagnose/config', () => { ); mockSetupDeps = createMockPluginSetup({ - elasticsearch: { - legacy: { client: { callAsInternalUser: jest.fn() } }, - }, router: httpSetup.createRouter(''), } as unknown) as any; @@ -58,6 +58,7 @@ describe('POST /diagnose/config', () => { }; core = await createMockReportingCore(config, mockSetupDeps); + mockEsClient = (await core.getEsClient()).asInternalUser as typeof mockEsClient; }); afterEach(async () => { @@ -65,15 +66,15 @@ describe('POST /diagnose/config', () => { }); it('returns a 200 by default when configured properly', async () => { - mockSetupDeps.elasticsearch.legacy.client.callAsInternalUser.mockImplementation(() => - Promise.resolve({ + mockEsClient.cluster.getSettings.mockResolvedValueOnce({ + body: { defaults: { http: { max_content_length: '100mb', }, }, - }) - ); + }, + } as any); registerDiagnoseConfig(core, mockLogger); await server.start(); @@ -94,15 +95,15 @@ describe('POST /diagnose/config', () => { it('returns a 200 with help text when not configured properly', async () => { config.get.mockImplementation(() => 10485760); - mockSetupDeps.elasticsearch.legacy.client.callAsInternalUser.mockImplementation(() => - Promise.resolve({ + mockEsClient.cluster.getSettings.mockResolvedValueOnce({ + body: { defaults: { http: { max_content_length: '5mb', }, }, - }) - ); + }, + } as any); registerDiagnoseConfig(core, mockLogger); await server.start(); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/config.ts b/x-pack/plugins/reporting/server/routes/diagnostic/config.ts index e3a01c464c36d..109849aa302f2 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/config.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/config.ts @@ -28,7 +28,7 @@ const numberToByteSizeValue = (value: number | ByteSizeValue) => { export const registerDiagnoseConfig = (reporting: ReportingCore, logger: Logger) => { const setupDeps = reporting.getPluginSetupDeps(); const userHandler = authorizedUserPreRoutingFactory(reporting); - const { router, elasticsearch } = setupDeps; + const { router } = setupDeps; router.post( { @@ -37,13 +37,13 @@ export const registerDiagnoseConfig = (reporting: ReportingCore, logger: Logger) }, userHandler(async (user, context, req, res) => { const warnings = []; - const { callAsInternalUser } = elasticsearch.legacy.client; + const { asInternalUser: elasticsearchClient } = await reporting.getEsClient(); const config = reporting.getConfig(); - const elasticClusterSettingsResponse = await callAsInternalUser('cluster.getSettings', { - includeDefaults: true, + const { body: clusterSettings } = await elasticsearchClient.cluster.getSettings({ + include_defaults: true, }); - const { persistent, transient, defaults: defaultSettings } = elasticClusterSettingsResponse; + const { persistent, transient, defaults: defaultSettings } = clusterSettings; const elasticClusterSettings = defaults({}, persistent, transient, defaultSettings); const elasticSearchMaxContent = get( diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index f6966a3b28ea9..0ce977e0a5431 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -6,8 +6,9 @@ */ import { UnwrapPromise } from '@kbn/utility-types'; +import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import { of } from 'rxjs'; -import sinon from 'sinon'; +import { ElasticsearchClient } from 'kibana/server'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '..'; @@ -24,8 +25,8 @@ describe('POST /api/reporting/generate', () => { let server: SetupServerReturn['server']; let httpSetup: SetupServerReturn['httpSetup']; let mockExportTypesRegistry: ExportTypesRegistry; - let callClusterStub: any; let core: ReportingCore; + let mockEsClient: DeeplyMockedKeys; const config = { get: jest.fn().mockImplementation((...args) => { @@ -55,12 +56,7 @@ describe('POST /api/reporting/generate', () => { () => ({}) ); - callClusterStub = sinon.stub().resolves({}); - const mockSetupDeps = createMockPluginSetup({ - elasticsearch: { - legacy: { client: { callAsInternalUser: callClusterStub } }, - }, security: { license: { isEnabled: () => true }, authc: { @@ -85,6 +81,9 @@ describe('POST /api/reporting/generate', () => { runTaskFnFactory: () => async () => ({ runParamsTest: { test2: 'yes' } } as any), }); core.getExportTypesRegistry = () => mockExportTypesRegistry; + + mockEsClient = (await core.getEsClient()).asInternalUser as typeof mockEsClient; + mockEsClient.index.mockResolvedValue({ body: {} } as any); }); afterEach(async () => { @@ -144,7 +143,7 @@ describe('POST /api/reporting/generate', () => { }); it('returns 500 if job handler throws an error', async () => { - callClusterStub.withArgs('index').rejects('silly'); + mockEsClient.index.mockRejectedValueOnce('silly'); registerJobGenerationRoutes(core, mockLogger); @@ -157,7 +156,7 @@ describe('POST /api/reporting/generate', () => { }); it(`returns 200 if job handler doesn't error`, async () => { - callClusterStub.withArgs('index').resolves({ _id: 'foo', _index: 'foo-index' }); + mockEsClient.index.mockResolvedValueOnce({ body: { _id: 'foo', _index: 'foo-index' } } as any); registerJobGenerationRoutes(core, mockLogger); await server.start(); diff --git a/x-pack/plugins/reporting/server/routes/jobs.test.ts b/x-pack/plugins/reporting/server/routes/jobs.test.ts index 706a8d5dad7dd..885fc701935fe 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.test.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.test.ts @@ -6,7 +6,9 @@ */ import { UnwrapPromise } from '@kbn/utility-types'; +import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import { of } from 'rxjs'; +import { ElasticsearchClient } from 'kibana/server'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '..'; @@ -29,6 +31,7 @@ describe('GET /api/reporting/jobs/download', () => { let httpSetup: SetupServerReturn['httpSetup']; let exportTypesRegistry: ExportTypesRegistry; let core: ReportingCore; + let mockEsClient: DeeplyMockedKeys; const config = createMockConfig(createMockConfigSchema()); const getHits = (...sources: any) => { @@ -47,9 +50,6 @@ describe('GET /api/reporting/jobs/download', () => { () => ({}) ); const mockSetupDeps = createMockPluginSetup({ - elasticsearch: { - legacy: { client: { callAsInternalUser: jest.fn() } }, - }, security: { license: { isEnabled: () => true, @@ -89,6 +89,8 @@ describe('GET /api/reporting/jobs/download', () => { validLicenses: ['basic', 'gold'], } as ExportTypeDefinition); core.getExportTypesRegistry = () => exportTypesRegistry; + + mockEsClient = (await core.getEsClient()).asInternalUser as typeof mockEsClient; }); afterEach(async () => { @@ -96,10 +98,7 @@ describe('GET /api/reporting/jobs/download', () => { }); it('fails on malformed download IDs', async () => { - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(getHits())), - }; + mockEsClient.search.mockResolvedValueOnce({ body: getHits() } as any); registerJobInfoRoutes(core); await server.start(); @@ -171,11 +170,7 @@ describe('GET /api/reporting/jobs/download', () => { }); it('returns 404 if job not found', async () => { - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(getHits())), - }; - + mockEsClient.search.mockResolvedValueOnce({ body: getHits() } as any); registerJobInfoRoutes(core); await server.start(); @@ -184,12 +179,9 @@ describe('GET /api/reporting/jobs/download', () => { }); it('returns a 401 if not a valid job type', async () => { - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest - .fn() - .mockReturnValue(Promise.resolve(getHits({ jobtype: 'invalidJobType' }))), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getHits({ jobtype: 'invalidJobType' }), + } as any); registerJobInfoRoutes(core); await server.start(); @@ -198,14 +190,9 @@ describe('GET /api/reporting/jobs/download', () => { }); it('when a job is incomplete', async () => { - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest - .fn() - .mockReturnValue( - Promise.resolve(getHits({ jobtype: 'unencodedJobType', status: 'pending' })) - ), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getHits({ jobtype: 'unencodedJobType', status: 'pending' }), + } as any); registerJobInfoRoutes(core); await server.start(); @@ -218,18 +205,13 @@ describe('GET /api/reporting/jobs/download', () => { }); it('when a job fails', async () => { - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue( - Promise.resolve( - getHits({ - jobtype: 'unencodedJobType', - status: 'failed', - output: { content: 'job failure message' }, - }) - ) - ), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getHits({ + jobtype: 'unencodedJobType', + status: 'failed', + output: { content: 'job failure message' }, + }), + } as any); registerJobInfoRoutes(core); await server.start(); @@ -243,7 +225,7 @@ describe('GET /api/reporting/jobs/download', () => { }); describe('successful downloads', () => { - const getCompleteHits = async ({ + const getCompleteHits = ({ jobType = 'unencodedJobType', outputContent = 'job output content', outputContentType = 'text/plain', @@ -260,11 +242,7 @@ describe('GET /api/reporting/jobs/download', () => { }; it('when a known job-type is complete', async () => { - const hits = getCompleteHits(); - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(hits)), - }; + mockEsClient.search.mockResolvedValueOnce({ body: getCompleteHits() } as any); registerJobInfoRoutes(core); await server.start(); @@ -276,11 +254,7 @@ describe('GET /api/reporting/jobs/download', () => { }); it('succeeds when security is not there or disabled', async () => { - const hits = getCompleteHits(); - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(hits)), - }; + mockEsClient.search.mockResolvedValueOnce({ body: getCompleteHits() } as any); // @ts-ignore core.pluginSetupDeps.security = null; @@ -297,14 +271,12 @@ describe('GET /api/reporting/jobs/download', () => { }); it(`doesn't encode output-content for non-specified job-types`, async () => { - const hits = getCompleteHits({ - jobType: 'unencodedJobType', - outputContent: 'test', - }); - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(hits)), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getCompleteHits({ + jobType: 'unencodedJobType', + outputContent: 'test', + }), + } as any); registerJobInfoRoutes(core); await server.start(); @@ -316,15 +288,13 @@ describe('GET /api/reporting/jobs/download', () => { }); it(`base64 encodes output content for configured jobTypes`, async () => { - const hits = getCompleteHits({ - jobType: 'base64EncodedJobType', - outputContent: 'test', - outputContentType: 'application/pdf', - }); - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(hits)), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getCompleteHits({ + jobType: 'base64EncodedJobType', + outputContent: 'test', + outputContentType: 'application/pdf', + }), + } as any); registerJobInfoRoutes(core); await server.start(); @@ -337,15 +307,13 @@ describe('GET /api/reporting/jobs/download', () => { }); it('refuses to return unknown content-types', async () => { - const hits = getCompleteHits({ - jobType: 'unencodedJobType', - outputContent: 'alert("all your base mine now");', - outputContentType: 'application/html', - }); - // @ts-ignore - core.pluginSetupDeps.elasticsearch.legacy.client = { - callAsInternalUser: jest.fn().mockReturnValue(Promise.resolve(hits)), - }; + mockEsClient.search.mockResolvedValueOnce({ + body: getCompleteHits({ + jobType: 'unencodedJobType', + outputContent: 'alert("all your base mine now");', + outputContentType: 'application/html', + }), + } as any); registerJobInfoRoutes(core); await server.start(); diff --git a/x-pack/plugins/reporting/server/routes/lib/jobs_query.ts b/x-pack/plugins/reporting/server/routes/lib/jobs_query.ts index 456c60e5c82e3..1db62f818216a 100644 --- a/x-pack/plugins/reporting/server/routes/lib/jobs_query.ts +++ b/x-pack/plugins/reporting/server/routes/lib/jobs_query.ts @@ -5,83 +5,59 @@ * 2.0. */ +import { UnwrapPromise } from '@kbn/utility-types'; import { i18n } from '@kbn/i18n'; -import { errors as elasticsearchErrors } from 'elasticsearch'; -import { get } from 'lodash'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { ElasticsearchClient } from 'src/core/server'; import { ReportingCore } from '../../'; import { ReportDocument } from '../../lib/store'; import { ReportingUser } from '../../types'; -const esErrors = elasticsearchErrors as Record; -const defaultSize = 10; - -// TODO: use SearchRequest from elasticsearch-client -interface QueryBody { - size?: number; - from?: number; - _source?: { - excludes: string[]; - }; - query: { - constant_score: { - filter: { - bool: { - must: Array>; - }; - }; - }; - }; -} +type SearchRequest = Required>[0]; interface GetOpts { includeContent?: boolean; } -// TODO: use SearchResult from elasticsearch-client -interface CountAggResult { - count: number; -} - +const defaultSize = 10; const getUsername = (user: ReportingUser) => (user ? user.username : false); -export function jobsQueryFactory(reportingCore: ReportingCore) { - const { elasticsearch } = reportingCore.getPluginSetupDeps(); - const { callAsInternalUser } = elasticsearch.legacy.client; - - function execQuery(queryType: string, body: QueryBody) { - const defaultBody: Record = { - search: { - _source: { - excludes: ['output.content'], - }, - sort: [{ created_at: { order: 'desc' } }], - size: defaultSize, - }, - }; +function getSearchBody(body: SearchRequest['body']): SearchRequest['body'] { + return { + _source: { + excludes: ['output.content'], + }, + sort: [{ created_at: { order: 'desc' } }], + size: defaultSize, + ...body, + }; +} +export function jobsQueryFactory(reportingCore: ReportingCore) { + function getIndex() { const config = reportingCore.getConfig(); - const index = config.get('index'); - const query = { - index: `${index}-*`, - body: Object.assign(defaultBody[queryType] || {}, body), - }; - - return callAsInternalUser(queryType, query).catch((err) => { - if (err instanceof esErrors['401']) return; - if (err instanceof esErrors['403']) return; - if (err instanceof esErrors['404']) return; - throw err; - }); + + return `${config.get('index')}-*`; } - type Result = number; + async function execQuery any>( + callback: T + ): Promise> | undefined> { + try { + const { asInternalUser: client } = await reportingCore.getEsClient(); + + return await callback(client); + } catch (error) { + if (error instanceof ResponseError && [401, 403, 404].includes(error.statusCode)) { + return; + } - function getHits(query: Promise) { - return query.then((res) => get(res, 'hits.hits', [])); + throw error; + } } return { - list( + async list( jobTypes: string[], user: ReportingUser, page = 0, @@ -89,32 +65,34 @@ export function jobsQueryFactory(reportingCore: ReportingCore) { jobIds: string[] | null ) { const username = getUsername(user); - const body: QueryBody = { + const body = getSearchBody({ size, from: size * page, query: { constant_score: { filter: { bool: { - must: [{ terms: { jobtype: jobTypes } }, { term: { created_by: username } }], + must: [ + { terms: { jobtype: jobTypes } }, + { term: { created_by: username } }, + ...(jobIds ? [{ ids: { values: jobIds } }] : []), + ], }, }, }, }, - }; + }); - if (jobIds) { - body.query.constant_score.filter.bool.must.push({ - ids: { values: jobIds }, - }); - } + const response = await execQuery((elasticsearchClient) => + elasticsearchClient.search({ body, index: getIndex() }) + ); - return getHits(execQuery('search', body)); + return response?.body.hits?.hits ?? []; }, - count(jobTypes: string[], user: ReportingUser) { + async count(jobTypes: string[], user: ReportingUser) { const username = getUsername(user); - const body: QueryBody = { + const body = { query: { constant_score: { filter: { @@ -126,17 +104,21 @@ export function jobsQueryFactory(reportingCore: ReportingCore) { }, }; - return execQuery('count', body).then((doc: CountAggResult) => { - if (!doc) return 0; - return doc.count; - }); + const response = await execQuery((elasticsearchClient) => + elasticsearchClient.count({ body, index: getIndex() }) + ); + + return response?.body.count ?? 0; }, - get(user: ReportingUser, id: string, opts: GetOpts = {}): Promise { - if (!id) return Promise.resolve(); - const username = getUsername(user); + async get(user: ReportingUser, id: string, opts: GetOpts = {}): Promise { + if (!id) { + return; + } - const body: QueryBody = { + const username = getUsername(user); + const body: SearchRequest['body'] = { + ...(opts.includeContent ? { _source: { excludes: [] } } : {}), query: { constant_score: { filter: { @@ -149,22 +131,23 @@ export function jobsQueryFactory(reportingCore: ReportingCore) { size: 1, }; - if (opts.includeContent) { - body._source = { - excludes: [], - }; + const response = await execQuery((elasticsearchClient) => + elasticsearchClient.search({ body, index: getIndex() }) + ); + + if (response?.body.hits?.hits?.length !== 1) { + return; } - return getHits(execQuery('search', body)).then((hits) => { - if (hits.length !== 1) return; - return hits[0]; - }); + return response.body.hits.hits[0] as ReportDocument; }, async delete(deleteIndex: string, id: string) { try { + const { asInternalUser: elasticsearchClient } = await reportingCore.getEsClient(); const query = { id, index: deleteIndex, refresh: true }; - return callAsInternalUser('delete', query); + + return await elasticsearchClient.delete(query); } catch (error) { throw new Error( i18n.translate('xpack.reporting.jobsQuery.deleteError', { diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index e42d87c50e118..952f801ba519d 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -37,7 +37,6 @@ import { createMockLevelLogger } from './create_mock_levellogger'; export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup => { return { features: featuresPluginMock.createSetup(), - elasticsearch: setupMock.elasticsearch || { legacy: { client: {} } }, basePath: { set: jest.fn() }, router: setupMock.router, security: setupMock.security, @@ -137,7 +136,7 @@ export const createMockReportingCore = async ( ) => { const mockReportingCore = ({ getConfig: () => config, - getElasticsearchService: () => setupDepsMock?.elasticsearch, + getEsClient: () => startDepsMock?.esClient, getDataService: () => startDepsMock?.data, } as unknown) as ReportingCore; From 106afd41b689fa0c3fc680d14b838d29650473e3 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Fri, 16 Apr 2021 10:40:30 +0200 Subject: [PATCH 10/25] [SavedObjects] Add aggregations support (#96292) * step 1 to add aggs in the find function of saved object * setp 2 - add specific unit test to aggs + fix bug found during integrations * step 3 - add security api_integration arounds aggs * fix types * unit test added for aggs_utils * add documentation * fix docs * review I * doc * try to fix test * add the new property to the saved object globaltype * fix types * delete old files * fix types + test api integration * type fix + test * Update src/core/server/saved_objects/types.ts Co-authored-by: Rudolf Meijering * review I * change our validation to match discussion with Pierre and Rudolph * Validate multiple items nested filter query through KueryNode * remove unused import * review + put back test * migrate added tests to new TS file * fix documentation * fix license header * move stuff * duplicating test mappings * rename some stuff * move ALL the things * cast to aggregation container * update generated doc * add deep nested validation * rewrite the whole validation mechanism * some cleanup * minor cleanup * update generated doc * adapt telemetry client * fix API integ tests * fix doc * TOTO-less * remove xpack tests * list supported / unsupported aggregations * typo fix * extract some validation function * fix indent * add some unit tests * adapt FTR assertions * update doc * fix doc * doc again * cleanup test names * improve tsdoc on validation functions * perf nit Co-authored-by: Xavier Mouligneau <189600+XavierM@users.noreply.github.com> Co-authored-by: Rudolf Meijering Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/api/saved-objects/find.asciidoc | 11 +- ...gin-core-public.savedobjectsclient.find.md | 2 +- ...a-plugin-core-public.savedobjectsclient.md | 2 +- ...dobjectsfindresponsepublic.aggregations.md | 11 + ...e-public.savedobjectsfindresponsepublic.md | 3 +- ...gin-core-server.savedobjectsclient.find.md | 4 +- ...r.savedobjectsfindresponse.aggregations.md | 11 + ...in-core-server.savedobjectsfindresponse.md | 3 +- ...core-server.savedobjectsrepository.find.md | 4 +- ...vedobjectsutils.createemptyfindresponse.md | 2 +- ...na-plugin-core-server.savedobjectsutils.md | 2 +- src/core/public/public.api.md | 8 +- .../saved_objects/saved_objects_client.ts | 14 +- src/core/server/saved_objects/routes/find.ts | 16 + .../aggregations/aggs_types/bucket_aggs.ts | 86 ++++ .../lib/aggregations/aggs_types/index.ts | 15 + .../aggregations/aggs_types/metrics_aggs.ts | 94 ++++ .../service/lib/aggregations/index.ts | 9 + .../lib/aggregations/validation.test.ts | 431 ++++++++++++++++++ .../service/lib/aggregations/validation.ts | 229 ++++++++++ .../lib/aggregations/validation_utils.test.ts | 148 ++++++ .../lib/aggregations/validation_utils.ts | 80 ++++ .../service/lib/filter_utils.test.ts | 69 ++- .../saved_objects/service/lib/filter_utils.ts | 12 +- .../saved_objects/service/lib/repository.ts | 36 +- .../server/saved_objects/service/lib/utils.ts | 4 +- .../service/saved_objects_client.ts | 7 +- src/core/server/saved_objects/types.ts | 22 + src/core/server/server.api.md | 12 +- .../server/telemetry_saved_objects_client.ts | 4 +- .../apis/saved_objects/find.ts | 69 +++ .../encrypted_saved_objects_client_wrapper.ts | 4 +- .../secure_saved_objects_client_wrapper.ts | 6 +- .../spaces_saved_objects_client.ts | 8 +- 34 files changed, 1369 insertions(+), 69 deletions(-) create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/index.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation.test.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts create mode 100644 src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts diff --git a/docs/api/saved-objects/find.asciidoc b/docs/api/saved-objects/find.asciidoc index c43b58d3aa989..f04aeb8420620 100644 --- a/docs/api/saved-objects/find.asciidoc +++ b/docs/api/saved-objects/find.asciidoc @@ -53,9 +53,14 @@ experimental[] Retrieve a paginated set of {kib} saved objects by various condit (Optional, object) Filters to objects that have a relationship with the type and ID combination. `filter`:: - (Optional, string) The filter is a KQL string with the caveat that if you filter with an attribute from your type saved object. - It should look like that savedObjectType.attributes.title: "myTitle". However, If you used a direct attribute of a saved object like `updatedAt`, - you will have to define your filter like that savedObjectType.updatedAt > 2018-12-22. + (Optional, string) The filter is a KQL string with the caveat that if you filter with an attribute from your saved object type, + it should look like that: `savedObjectType.attributes.title: "myTitle"`. However, If you use a root attribute of a saved + object such as `updated_at`, you will have to define your filter like that: `savedObjectType.updated_at > 2018-12-22`. + +`aggs`:: + (Optional, string) **experimental** An aggregation structure, serialized as a string. The field format is similar to `filter`, meaning + that to use a saved object type attribute in the aggregation, the `savedObjectType.attributes.title`: "myTitle"` format + must be used. For root fields, the syntax is `savedObjectType.rootField` NOTE: As objects change in {kib}, the results on each page of the response also change. Use the find API for traditional paginated results, but avoid using it to export large amounts of data. diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md index ddd8b207e3d78..fc9652b96450f 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.find.md @@ -9,5 +9,5 @@ Search for objects Signature: ```typescript -find: (options: SavedObjectsFindOptions) => Promise>; +find: (options: SavedObjectsFindOptions) => Promise>; ``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md index 6e53b169b8bed..1ec756f8d743d 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsclient.md @@ -24,7 +24,7 @@ The constructor for this class is marked as internal. Third-party code should no | [bulkGet](./kibana-plugin-core-public.savedobjectsclient.bulkget.md) | | (objects?: Array<{
id: string;
type: string;
}>) => Promise<SavedObjectsBatchResponse<unknown>> | Returns an array of objects by id | | [create](./kibana-plugin-core-public.savedobjectsclient.create.md) | | <T = unknown>(type: string, attributes: T, options?: SavedObjectsCreateOptions) => Promise<SimpleSavedObject<T>> | Persists an object | | [delete](./kibana-plugin-core-public.savedobjectsclient.delete.md) | | (type: string, id: string, options?: SavedObjectsDeleteOptions | undefined) => ReturnType<SavedObjectsApi['delete']> | Deletes an object | -| [find](./kibana-plugin-core-public.savedobjectsclient.find.md) | | <T = unknown>(options: SavedObjectsFindOptions) => Promise<SavedObjectsFindResponsePublic<T>> | Search for objects | +| [find](./kibana-plugin-core-public.savedobjectsclient.find.md) | | <T = unknown, A = unknown>(options: SavedObjectsFindOptions) => Promise<SavedObjectsFindResponsePublic<T, unknown>> | Search for objects | | [get](./kibana-plugin-core-public.savedobjectsclient.get.md) | | <T = unknown>(type: string, id: string) => Promise<SimpleSavedObject<T>> | Fetches a single object | ## Methods diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md new file mode 100644 index 0000000000000..14401b02f25c7 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindResponsePublic](./kibana-plugin-core-public.savedobjectsfindresponsepublic.md) > [aggregations](./kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md) + +## SavedObjectsFindResponsePublic.aggregations property + +Signature: + +```typescript +aggregations?: A; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md index 7d75878041264..6f2276194f054 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.md @@ -11,13 +11,14 @@ Return type of the Saved Objects `find()` method. Signature: ```typescript -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse +export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse ``` ## Properties | Property | Type | Description | | --- | --- | --- | +| [aggregations](./kibana-plugin-core-public.savedobjectsfindresponsepublic.aggregations.md) | A | | | [page](./kibana-plugin-core-public.savedobjectsfindresponsepublic.page.md) | number | | | [perPage](./kibana-plugin-core-public.savedobjectsfindresponsepublic.perpage.md) | number | | | [total](./kibana-plugin-core-public.savedobjectsfindresponsepublic.total.md) | number | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md index 9a4c3df5d2d92..56d76125108d1 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.find.md @@ -9,7 +9,7 @@ Find all SavedObjects matching the search query Signature: ```typescript -find(options: SavedObjectsFindOptions): Promise>; +find(options: SavedObjectsFindOptions): Promise>; ``` ## Parameters @@ -20,5 +20,5 @@ find(options: SavedObjectsFindOptions): PromiseReturns: -`Promise>` +`Promise>` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md new file mode 100644 index 0000000000000..17a899f4c8280 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResponse](./kibana-plugin-core-server.savedobjectsfindresponse.md) > [aggregations](./kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md) + +## SavedObjectsFindResponse.aggregations property + +Signature: + +```typescript +aggregations?: A; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md index fd56e8ce40e24..8176baf44acbd 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md @@ -11,13 +11,14 @@ Return type of the Saved Objects `find()` method. Signature: ```typescript -export interface SavedObjectsFindResponse +export interface SavedObjectsFindResponse ``` ## Properties | Property | Type | Description | | --- | --- | --- | +| [aggregations](./kibana-plugin-core-server.savedobjectsfindresponse.aggregations.md) | A | | | [page](./kibana-plugin-core-server.savedobjectsfindresponse.page.md) | number | | | [per\_page](./kibana-plugin-core-server.savedobjectsfindresponse.per_page.md) | number | | | [pit\_id](./kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md index d3e93e7af2aa0..5c823b7567918 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.find.md @@ -7,7 +7,7 @@ Signature: ```typescript -find(options: SavedObjectsFindOptions): Promise>; +find(options: SavedObjectsFindOptions): Promise>; ``` ## Parameters @@ -18,7 +18,7 @@ find(options: SavedObjectsFindOptions): PromiseReturns: -`Promise>` +`Promise>` {promise} - { saved\_objects: \[{ id, type, version, attributes }\], total, per\_page, page } diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md index 40e865cb02ce8..23cbebf22aa21 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md @@ -9,5 +9,5 @@ Creates an empty response for a find operation. This is only intended to be used Signature: ```typescript -static createEmptyFindResponse: ({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; +static createEmptyFindResponse: ({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.md index 8c787364c4cbe..0148621e757b7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsutils.md @@ -15,7 +15,7 @@ export declare class SavedObjectsUtils | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [createEmptyFindResponse](./kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md) | static | <T>({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse<T> | Creates an empty response for a find operation. This is only intended to be used by saved objects client wrappers. | +| [createEmptyFindResponse](./kibana-plugin-core-server.savedobjectsutils.createemptyfindresponse.md) | static | <T, A>({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse<T, A> | Creates an empty response for a find operation. This is only intended to be used by saved objects client wrappers. | | [namespaceIdToString](./kibana-plugin-core-server.savedobjectsutils.namespaceidtostring.md) | static | (namespace?: string | undefined) => string | Converts a given saved object namespace ID to its string representation. All namespace IDs have an identical string representation, with the exception of the undefined namespace ID (which has a namespace string of 'default'). | | [namespaceStringToId](./kibana-plugin-core-server.savedobjectsutils.namespacestringtoid.md) | static | (namespace: string) => string | undefined | Converts a given saved object namespace string to its ID representation. All namespace strings have an identical ID representation, with the exception of the 'default' namespace string (which has a namespace ID of undefined). | diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 8c1753c2cabab..18133ebec3353 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1224,7 +1224,7 @@ export class SavedObjectsClient { // Warning: (ae-forgotten-export) The symbol "SavedObjectsClientContract" needs to be exported by the entry point index.d.ts delete: (type: string, id: string, options?: SavedObjectsDeleteOptions | undefined) => ReturnType; // Warning: (ae-forgotten-export) The symbol "SavedObjectsFindOptions" needs to be exported by the entry point index.d.ts - find: (options: SavedObjectsFindOptions_2) => Promise>; + find: (options: SavedObjectsFindOptions_2) => Promise>; get: (type: string, id: string) => Promise>; update(type: string, id: string, attributes: T, { version, migrationVersion, references }?: SavedObjectsUpdateOptions): Promise>; } @@ -1244,6 +1244,8 @@ export interface SavedObjectsCreateOptions { // @public (undocumented) export interface SavedObjectsFindOptions { + // @alpha + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts @@ -1284,7 +1286,9 @@ export interface SavedObjectsFindOptionsReference { } // @public -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { +export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { + // (undocumented) + aggregations?: A; // (undocumented) page: number; // (undocumented) diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 44466025de7e3..782ffa6897048 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -103,7 +103,9 @@ export interface SavedObjectsDeleteOptions { * * @public */ -export interface SavedObjectsFindResponsePublic extends SavedObjectsBatchResponse { +export interface SavedObjectsFindResponsePublic + extends SavedObjectsBatchResponse { + aggregations?: A; total: number; perPage: number; page: number; @@ -310,7 +312,7 @@ export class SavedObjectsClient { * @property {object} [options.hasReference] - { type, id } * @returns A find result with objects matching the specified search. */ - public find = ( + public find = ( options: SavedObjectsFindOptions ): Promise> => { const path = this.getPath(['_find']); @@ -326,6 +328,7 @@ export class SavedObjectsClient { sortField: 'sort_field', type: 'type', filter: 'filter', + aggs: 'aggs', namespaces: 'namespaces', preference: 'preference', }; @@ -342,6 +345,12 @@ export class SavedObjectsClient { query.has_reference = JSON.stringify(query.has_reference); } + // `aggs` is a structured object. we need to stringify it before sending it, as `fetch` + // is not doing it implicitly. + if (query.aggs) { + query.aggs = JSON.stringify(query.aggs); + } + const request: ReturnType = this.savedObjectsFetch(path, { method: 'GET', query, @@ -349,6 +358,7 @@ export class SavedObjectsClient { return request.then((resp) => { return renameKeys( { + aggregations: 'aggregations', saved_objects: 'savedObjects', total: 'total', per_page: 'perPage', diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 6ba23747cf374..d21039db30e5f 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -44,6 +44,7 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen has_reference_operator: searchOperatorSchema, fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), filter: schema.maybe(schema.string()), + aggs: schema.maybe(schema.string()), namespaces: schema.maybe( schema.oneOf([schema.string(), schema.arrayOf(schema.string())]) ), @@ -59,6 +60,20 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen const usageStatsClient = coreUsageData.getClient(); usageStatsClient.incrementSavedObjectsFind({ request: req }).catch(() => {}); + // manually validation to avoid using JSON.parse twice + let aggs; + if (query.aggs) { + try { + aggs = JSON.parse(query.aggs); + } catch (e) { + return res.badRequest({ + body: { + message: 'invalid aggs value', + }, + }); + } + } + const result = await context.core.savedObjects.client.find({ perPage: query.per_page, page: query.page, @@ -72,6 +87,7 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen hasReferenceOperator: query.has_reference_operator, fields: typeof query.fields === 'string' ? [query.fields] : query.fields, filter: query.filter, + aggs, namespaces, }); diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts new file mode 100644 index 0000000000000..1508cab69a048 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema as s, ObjectType } from '@kbn/config-schema'; + +/** + * Schemas for the Bucket aggregations. + * + * Currently supported: + * - filter + * - histogram + * - terms + * + * Not implemented: + * - adjacency_matrix + * - auto_date_histogram + * - children + * - composite + * - date_histogram + * - date_range + * - diversified_sampler + * - filters + * - geo_distance + * - geohash_grid + * - geotile_grid + * - global + * - ip_range + * - missing + * - multi_terms + * - nested + * - parent + * - range + * - rare_terms + * - reverse_nested + * - sampler + * - significant_terms + * - significant_text + * - variable_width_histogram + */ +export const bucketAggsSchemas: Record = { + filter: s.object({ + term: s.recordOf(s.string(), s.oneOf([s.string(), s.boolean(), s.number()])), + }), + histogram: s.object({ + field: s.maybe(s.string()), + interval: s.maybe(s.number()), + min_doc_count: s.maybe(s.number()), + extended_bounds: s.maybe( + s.object({ + min: s.number(), + max: s.number(), + }) + ), + hard_bounds: s.maybe( + s.object({ + min: s.number(), + max: s.number(), + }) + ), + missing: s.maybe(s.number()), + keyed: s.maybe(s.boolean()), + order: s.maybe( + s.object({ + _count: s.string(), + _key: s.string(), + }) + ), + }), + terms: s.object({ + field: s.maybe(s.string()), + collect_mode: s.maybe(s.string()), + exclude: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + include: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + execution_hint: s.maybe(s.string()), + missing: s.maybe(s.number()), + min_doc_count: s.maybe(s.number()), + size: s.maybe(s.number()), + show_term_doc_count_error: s.maybe(s.boolean()), + order: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + }), +}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts new file mode 100644 index 0000000000000..7967fad0185fb --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { bucketAggsSchemas } from './bucket_aggs'; +import { metricsAggsSchemas } from './metrics_aggs'; + +export const aggregationSchemas = { + ...metricsAggsSchemas, + ...bucketAggsSchemas, +}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts new file mode 100644 index 0000000000000..c05ae67cd2164 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema as s, ObjectType } from '@kbn/config-schema'; + +/** + * Schemas for the metrics Aggregations + * + * Currently supported: + * - avg + * - cardinality + * - min + * - max + * - sum + * - top_hits + * - weighted_avg + * + * Not implemented: + * - boxplot + * - extended_stats + * - geo_bounds + * - geo_centroid + * - geo_line + * - matrix_stats + * - median_absolute_deviation + * - percentile_ranks + * - percentiles + * - rate + * - scripted_metric + * - stats + * - string_stats + * - t_test + * - value_count + */ +export const metricsAggsSchemas: Record = { + avg: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + cardinality: s.object({ + field: s.maybe(s.string()), + precision_threshold: s.maybe(s.number()), + rehash: s.maybe(s.boolean()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + min: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + format: s.maybe(s.string()), + }), + max: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + format: s.maybe(s.string()), + }), + sum: s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + }), + top_hits: s.object({ + explain: s.maybe(s.boolean()), + docvalue_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), + from: s.maybe(s.number()), + size: s.maybe(s.number()), + sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + seq_no_primary_term: s.maybe(s.boolean()), + version: s.maybe(s.boolean()), + track_scores: s.maybe(s.boolean()), + highlight: s.maybe(s.any()), + _source: s.maybe(s.oneOf([s.boolean(), s.string(), s.arrayOf(s.string())])), + }), + weighted_avg: s.object({ + format: s.maybe(s.string()), + value_type: s.maybe(s.string()), + value: s.maybe( + s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.number()), + }) + ), + weight: s.maybe( + s.object({ + field: s.maybe(s.string()), + missing: s.maybe(s.number()), + }) + ), + }), +}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/index.ts b/src/core/server/saved_objects/service/lib/aggregations/index.ts new file mode 100644 index 0000000000000..f71d3e8daea9d --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { validateAndConvertAggregations } from './validation'; diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts new file mode 100644 index 0000000000000..8a7c1c3719eb0 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.test.ts @@ -0,0 +1,431 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import { validateAndConvertAggregations } from './validation'; + +type AggsMap = Record; + +const mockMappings = { + properties: { + updated_at: { + type: 'date', + }, + foo: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + bytes: { + type: 'number', + }, + }, + }, + bean: { + properties: { + canned: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + alert: { + properties: { + actions: { + type: 'nested', + properties: { + group: { + type: 'keyword', + }, + actionRef: { + type: 'keyword', + }, + actionTypeId: { + type: 'keyword', + }, + params: { + enabled: false, + type: 'object', + }, + }, + }, + params: { + type: 'flattened', + }, + }, + }, + }, +}; + +describe('validateAndConvertAggregations', () => { + it('validates a simple aggregations', () => { + expect( + validateAndConvertAggregations( + ['foo'], + { aggName: { max: { field: 'foo.attributes.bytes' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + max: { + field: 'foo.bytes', + }, + }, + }); + }); + + it('validates a nested field in simple aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { aggName: { cardinality: { field: 'alert.attributes.actions.group' } } }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', + }, + }, + }); + }); + + it('validates a nested aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { + aggName: { + cardinality: { + field: 'alert.attributes.actions.group', + }, + aggs: { + aggName: { + max: { field: 'alert.attributes.actions.group' }, + }, + }, + }, + }, + mockMappings + ) + ).toEqual({ + aggName: { + cardinality: { + field: 'alert.actions.group', + }, + aggs: { + aggName: { + max: { + field: 'alert.actions.group', + }, + }, + }, + }, + }); + }); + + it('validates a deeply nested aggregations', () => { + expect( + validateAndConvertAggregations( + ['alert'], + { + first: { + cardinality: { + field: 'alert.attributes.actions.group', + }, + aggs: { + second: { + max: { field: 'alert.attributes.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.attributes.actions.actionTypeId', + }, + }, + }, + }, + }, + }, + }, + mockMappings + ) + ).toEqual({ + first: { + cardinality: { + field: 'alert.actions.group', + }, + aggs: { + second: { + max: { field: 'alert.actions.group' }, + aggs: { + third: { + min: { + field: 'alert.actions.actionTypeId', + }, + }, + }, + }, + }, + }, + }); + }); + + it('rewrites type attributes when valid', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.group', + missing: 10, + }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + avg: { + field: 'alert.actions.group', + missing: 10, + }, + }, + }); + }); + + it('rewrites root attributes when valid', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.updated_at', + missing: 10, + }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + avg: { + field: 'updated_at', + missing: 10, + }, + }, + }); + }); + + it('throws an error when the `field` name is not using attributes path', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.actions.group', + missing: 10, + }, + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.actions.group"` + ); + }); + + it('throws an error when the `field` name is referencing an invalid field', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.attributes.actions.non_existing', + missing: 10, + }, + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.attributes.actions.non_existing"` + ); + }); + + it('throws an error when the attribute path is referencing an invalid root field', () => { + const aggregations: AggsMap = { + average: { + avg: { + field: 'alert.bad_root', + missing: 10, + }, + }, + }; + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[average.avg.field] Invalid attribute path: alert.bad_root"` + ); + }); + + it('rewrites the `field` name even when nested', () => { + const aggregations: AggsMap = { + average: { + weighted_avg: { + value: { + field: 'alert.attributes.actions.group', + missing: 10, + }, + weight: { + field: 'alert.attributes.actions.actionRef', + }, + }, + }, + }; + expect(validateAndConvertAggregations(['alert'], aggregations, mockMappings)).toEqual({ + average: { + weighted_avg: { + value: { + field: 'alert.actions.group', + missing: 10, + }, + weight: { + field: 'alert.actions.actionRef', + }, + }, + }, + }); + }); + + it('rewrites the entries of a filter term record', () => { + const aggregations: AggsMap = { + myFilter: { + filter: { + term: { + 'foo.attributes.description': 'hello', + 'foo.attributes.bytes': 10, + }, + }, + }, + }; + expect(validateAndConvertAggregations(['foo'], aggregations, mockMappings)).toEqual({ + myFilter: { + filter: { + term: { 'foo.description': 'hello', 'foo.bytes': 10 }, + }, + }, + }); + }); + + it('throws an error when referencing non-allowed types', () => { + const aggregations: AggsMap = { + myFilter: { + max: { + field: 'foo.attributes.bytes', + }, + }, + }; + + expect(() => { + validateAndConvertAggregations(['alert'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[myFilter.max.field] Invalid attribute path: foo.attributes.bytes"` + ); + }); + + it('throws an error when an attributes is not respecting its schema definition', () => { + const aggregations: AggsMap = { + someAgg: { + terms: { + missing: 'expecting a number', + }, + }, + }; + + expect(() => + validateAndConvertAggregations(['alert'], aggregations, mockMappings) + ).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.terms.missing]: expected value of type [number] but got [string]"` + ); + }); + + it('throws an error when trying to validate an unknown aggregation type', () => { + const aggregations: AggsMap = { + someAgg: { + auto_date_histogram: { + field: 'foo.attributes.bytes', + }, + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.auto_date_histogram] auto_date_histogram aggregation is not valid (or not registered yet)"` + ); + }); + + it('throws an error when a child aggregation is unknown', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + }, + aggs: { + unknownAgg: { + cumulative_cardinality: { + format: 'format', + }, + }, + }, + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.aggs.unknownAgg.cumulative_cardinality] cumulative_cardinality aggregation is not valid (or not registered yet)"` + ); + }); + + it('throws an error when using a script attribute', () => { + const aggregations: AggsMap = { + someAgg: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', + }, + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.max.script]: definition for this key is missing"` + ); + }); + + it('throws an error when using a script attribute in a nested aggregation', () => { + const aggregations: AggsMap = { + someAgg: { + min: { + field: 'foo.attributes.bytes', + }, + aggs: { + nested: { + max: { + field: 'foo.attributes.bytes', + script: 'This is a bad script', + }, + }, + }, + }, + }; + + expect(() => { + validateAndConvertAggregations(['foo'], aggregations, mockMappings); + }).toThrowErrorMatchingInlineSnapshot( + `"[someAgg.aggs.nested.max.script]: definition for this key is missing"` + ); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation.ts b/src/core/server/saved_objects/service/lib/aggregations/validation.ts new file mode 100644 index 0000000000000..a2fd392183132 --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import { ObjectType } from '@kbn/config-schema'; +import { isPlainObject } from 'lodash'; + +import { IndexMapping } from '../../../mappings'; +import { + isObjectTypeAttribute, + rewriteObjectTypeAttribute, + isRootLevelAttribute, + rewriteRootLevelAttribute, +} from './validation_utils'; +import { aggregationSchemas } from './aggs_types'; + +const aggregationKeys = ['aggs', 'aggregations']; + +interface ValidationContext { + allowedTypes: string[]; + indexMapping: IndexMapping; + currentPath: string[]; +} + +/** + * Validate an aggregation structure against the declared mappings and + * aggregation schemas, and rewrite the attribute fields using the KQL-like syntax + * - `{type}.attributes.{attribute}` to `{type}.{attribute}` + * - `{type}.{rootField}` to `{rootField}` + * + * throws on the first validation error if any is encountered. + */ +export const validateAndConvertAggregations = ( + allowedTypes: string[], + aggs: Record, + indexMapping: IndexMapping +): Record => { + return validateAggregations(aggs, { + allowedTypes, + indexMapping, + currentPath: [], + }); +}; + +/** + * Validate a record of aggregation containers, + * Which can either be the root level aggregations (`SearchRequest.body.aggs`) + * Or a nested record of aggregation (`SearchRequest.body.aggs.myAggregation.aggs`) + */ +const validateAggregations = ( + aggregations: Record, + context: ValidationContext +) => { + return Object.entries(aggregations).reduce((memo, [aggrName, aggrContainer]) => { + memo[aggrName] = validateAggregation(aggrContainer, childContext(context, aggrName)); + return memo; + }, {} as Record); +}; + +/** + * Validate an aggregation container, e.g an entry of `SearchRequest.body.aggs`, or + * from a nested aggregation record, including its potential nested aggregations. + */ +const validateAggregation = ( + aggregation: estypes.AggregationContainer, + context: ValidationContext +) => { + const container = validateAggregationContainer(aggregation, context); + + if (aggregation.aggregations) { + container.aggregations = validateAggregations( + aggregation.aggregations, + childContext(context, 'aggregations') + ); + } + if (aggregation.aggs) { + container.aggs = validateAggregations(aggregation.aggs, childContext(context, 'aggs')); + } + + return container; +}; + +/** + * Validates root-level aggregation of given aggregation container + * (ignoring its nested aggregations) + */ +const validateAggregationContainer = ( + container: estypes.AggregationContainer, + context: ValidationContext +) => { + return Object.entries(container).reduce((memo, [aggName, aggregation]) => { + if (aggregationKeys.includes(aggName)) { + return memo; + } + return { + ...memo, + [aggName]: validateAggregationType(aggName, aggregation, childContext(context, aggName)), + }; + }, {} as estypes.AggregationContainer); +}; + +const validateAggregationType = ( + aggregationType: string, + aggregation: Record, + context: ValidationContext +) => { + const aggregationSchema = aggregationSchemas[aggregationType]; + if (!aggregationSchema) { + throw new Error( + `[${context.currentPath.join( + '.' + )}] ${aggregationType} aggregation is not valid (or not registered yet)` + ); + } + + validateAggregationStructure(aggregationSchema, aggregation, context); + return validateAndRewriteFieldAttributes(aggregation, context); +}; + +/** + * Validate an aggregation structure against its declared schema. + */ +const validateAggregationStructure = ( + schema: ObjectType, + aggObject: unknown, + context: ValidationContext +) => { + return schema.validate(aggObject, {}, context.currentPath.join('.')); +}; + +/** + * List of fields that have an attribute path as value + * + * @example + * ```ts + * avg: { + * field: 'alert.attributes.actions.group', + * }, + * ``` + */ +const attributeFields = ['field']; +/** + * List of fields that have a Record as value + * + * @example + * ```ts + * filter: { + * term: { + * 'alert.attributes.actions.group': 'value' + * }, + * }, + * ``` + */ +const attributeMaps = ['term']; + +const validateAndRewriteFieldAttributes = ( + aggregation: Record, + context: ValidationContext +) => { + return recursiveRewrite(aggregation, context, []); +}; + +const recursiveRewrite = ( + currentLevel: Record, + context: ValidationContext, + parents: string[] +): Record => { + return Object.entries(currentLevel).reduce((memo, [key, value]) => { + const rewriteKey = isAttributeKey(parents); + const rewriteValue = isAttributeValue(key, value); + + const nestedContext = childContext(context, key); + const newKey = rewriteKey ? validateAndRewriteAttributePath(key, nestedContext) : key; + const newValue = rewriteValue + ? validateAndRewriteAttributePath(value, nestedContext) + : isPlainObject(value) + ? recursiveRewrite(value, nestedContext, [...parents, key]) + : value; + + return { + ...memo, + [newKey]: newValue, + }; + }, {}); +}; + +const childContext = (context: ValidationContext, path: string): ValidationContext => { + return { + ...context, + currentPath: [...context.currentPath, path], + }; +}; + +const lastParent = (parents: string[]) => { + if (parents.length) { + return parents[parents.length - 1]; + } + return undefined; +}; + +const isAttributeKey = (parents: string[]) => { + const last = lastParent(parents); + if (last) { + return attributeMaps.includes(last); + } + return false; +}; + +const isAttributeValue = (fieldName: string, fieldValue: unknown): boolean => { + return attributeFields.includes(fieldName) && typeof fieldValue === 'string'; +}; + +const validateAndRewriteAttributePath = ( + attributePath: string, + { allowedTypes, indexMapping, currentPath }: ValidationContext +) => { + if (isRootLevelAttribute(attributePath, indexMapping, allowedTypes)) { + return rewriteRootLevelAttribute(attributePath); + } + if (isObjectTypeAttribute(attributePath, indexMapping, allowedTypes)) { + return rewriteObjectTypeAttribute(attributePath); + } + throw new Error(`[${currentPath.join('.')}] Invalid attribute path: ${attributePath}`); +}; diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts new file mode 100644 index 0000000000000..25c3aea474ece --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndexMapping } from '../../../mappings'; +import { + isRootLevelAttribute, + rewriteRootLevelAttribute, + isObjectTypeAttribute, + rewriteObjectTypeAttribute, +} from './validation_utils'; + +const mockMappings: IndexMapping = { + properties: { + updated_at: { + type: 'date', + }, + foo: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + bytes: { + type: 'number', + }, + }, + }, + bean: { + properties: { + canned: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + alert: { + properties: { + actions: { + type: 'nested', + properties: { + group: { + type: 'keyword', + }, + actionRef: { + type: 'keyword', + }, + actionTypeId: { + type: 'keyword', + }, + params: { + enabled: false, + type: 'object', + }, + }, + }, + params: { + type: 'flattened', + }, + }, + }, + }, +}; + +describe('isRootLevelAttribute', () => { + it('returns true when referring to a path to a valid root level field', () => { + expect(isRootLevelAttribute('foo.updated_at', mockMappings, ['foo'])).toBe(true); + }); + it('returns false when referring to a direct path to a valid root level field', () => { + expect(isRootLevelAttribute('updated_at', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a unknown root level field', () => { + expect(isRootLevelAttribute('foo.not_present', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to an existing nested field', () => { + expect(isRootLevelAttribute('foo.properties.title', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a valid root level field of an unknown type', () => { + expect(isRootLevelAttribute('bar.updated_at', mockMappings, ['foo'])).toBe(false); + }); + it('returns false when referring to a path to a valid root level type field', () => { + expect(isRootLevelAttribute('foo.foo', mockMappings, ['foo'])).toBe(false); + }); +}); + +describe('rewriteRootLevelAttribute', () => { + it('rewrites the attribute path to strip the type', () => { + expect(rewriteRootLevelAttribute('foo.references')).toEqual('references'); + }); + it('does not handle real root level path', () => { + expect(rewriteRootLevelAttribute('references')).not.toEqual('references'); + }); +}); + +describe('isObjectTypeAttribute', () => { + it('return true if attribute path is valid', () => { + expect(isObjectTypeAttribute('foo.attributes.description', mockMappings, ['foo'])).toEqual( + true + ); + }); + + it('return true for nested attributes', () => { + expect(isObjectTypeAttribute('bean.attributes.canned.text', mockMappings, ['bean'])).toEqual( + true + ); + }); + + it('return false if attribute path points to an invalid type', () => { + expect(isObjectTypeAttribute('foo.attributes.description', mockMappings, ['bean'])).toEqual( + false + ); + }); + + it('returns false if attribute path refers to a type', () => { + expect(isObjectTypeAttribute('bean', mockMappings, ['bean'])).toEqual(false); + }); + + it('Return error if key does not match SO attribute structure', () => { + expect(isObjectTypeAttribute('bean.canned.text', mockMappings, ['bean'])).toEqual(false); + }); + + it('Return false if key matches nested type attribute parent', () => { + expect(isObjectTypeAttribute('alert.actions', mockMappings, ['alert'])).toEqual(false); + }); + + it('returns false if path refers to a non-existent attribute', () => { + expect(isObjectTypeAttribute('bean.attributes.red', mockMappings, ['bean'])).toEqual(false); + }); +}); + +describe('rewriteObjectTypeAttribute', () => { + it('rewrites the attribute path to strip the type', () => { + expect(rewriteObjectTypeAttribute('foo.attributes.prop')).toEqual('foo.prop'); + }); + it('returns invalid input unchanged', () => { + expect(rewriteObjectTypeAttribute('foo.references')).toEqual('foo.references'); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts new file mode 100644 index 0000000000000..f817497e3759e --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/validation_utils.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndexMapping } from '../../../mappings'; +import { fieldDefined, hasFilterKeyError } from '../filter_utils'; + +/** + * Returns true if the given attribute path is a valid root level SO attribute path + * + * @example + * ```ts + * isRootLevelAttribute('myType.updated_at', indexMapping, ['myType']}) + * // => true + * ``` + */ +export const isRootLevelAttribute = ( + attributePath: string, + indexMapping: IndexMapping, + allowedTypes: string[] +): boolean => { + const splits = attributePath.split('.'); + if (splits.length !== 2) { + return false; + } + + const [type, fieldName] = splits; + if (allowedTypes.includes(fieldName)) { + return false; + } + return allowedTypes.includes(type) && fieldDefined(indexMapping, fieldName); +}; + +/** + * Rewrites a root level attribute path to strip the type + * + * @example + * ```ts + * rewriteRootLevelAttribute('myType.updated_at') + * // => 'updated_at' + * ``` + */ +export const rewriteRootLevelAttribute = (attributePath: string) => { + return attributePath.split('.')[1]; +}; + +/** + * Returns true if the given attribute path is a valid object type level SO attribute path + * + * @example + * ```ts + * isObjectTypeAttribute('myType.attributes.someField', indexMapping, ['myType']}) + * // => true + * ``` + */ +export const isObjectTypeAttribute = ( + attributePath: string, + indexMapping: IndexMapping, + allowedTypes: string[] +): boolean => { + const error = hasFilterKeyError(attributePath, allowedTypes, indexMapping); + return error == null; +}; + +/** + * Rewrites a object type attribute path to strip the type + * + * @example + * ```ts + * rewriteObjectTypeAttribute('myType.attributes.foo') + * // => 'myType.foo' + * ``` + */ +export const rewriteObjectTypeAttribute = (attributePath: string) => { + return attributePath.replace('.attributes', ''); +}; diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index b50326627cf09..956a60b23809d 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -18,7 +18,7 @@ import { const mockMappings = { properties: { - updatedAt: { + updated_at: { type: 'date', }, foo: { @@ -123,12 +123,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo'], - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '(type: foo and updatedAt: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' + '(type: foo and updated_at: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' ) ); }); @@ -137,12 +137,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo', 'bar'], - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '(type: foo and updatedAt: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' + '(type: foo and updated_at: 5678654567) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or foo.description :*)' ) ); }); @@ -151,12 +151,12 @@ describe('Filter Utils', () => { expect( validateConvertFilterToKueryNode( ['foo', 'bar'], - '(bar.updatedAt: 5678654567 OR foo.updatedAt: 5678654567) and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or bar.attributes.description :*)', + '(bar.updated_at: 5678654567 OR foo.updated_at: 5678654567) and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or bar.attributes.description :*)', mockMappings ) ).toEqual( esKuery.fromKueryExpression( - '((type: bar and updatedAt: 5678654567) or (type: foo and updatedAt: 5678654567)) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or bar.description :*)' + '((type: bar and updated_at: 5678654567) or (type: foo and updated_at: 5678654567)) and foo.bytes > 1000 and foo.bytes < 8000 and foo.title: "best" and (foo.description: t* or bar.description :*)' ) ); }); @@ -181,11 +181,11 @@ describe('Filter Utils', () => { expect(() => { validateConvertFilterToKueryNode( ['foo', 'bar'], - 'updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', + 'updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)', mockMappings ); }).toThrowErrorMatchingInlineSnapshot( - `"This key 'updatedAt' need to be wrapped by a saved object type like foo,bar: Bad Request"` + `"This key 'updated_at' need to be wrapped by a saved object type like foo,bar: Bad Request"` ); }); @@ -200,7 +200,7 @@ describe('Filter Utils', () => { test('Validate filter query through KueryNode - happy path', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -211,7 +211,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: null, isSavedObjectAttr: true, - key: 'foo.updatedAt', + key: 'foo.updated_at', type: 'foo', }, { @@ -275,7 +275,7 @@ describe('Filter Utils', () => { test('Return Error if key is not wrapper by a saved object type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -284,9 +284,9 @@ describe('Filter Utils', () => { expect(validationObject).toEqual([ { astPath: 'arguments.0', - error: "This key 'updatedAt' need to be wrapped by a saved object type like foo", + error: "This key 'updated_at' need to be wrapped by a saved object type like foo", isSavedObjectAttr: true, - key: 'updatedAt', + key: 'updated_at', type: null, }, { @@ -330,7 +330,7 @@ describe('Filter Utils', () => { test('Return Error if key of a saved object type is not wrapped with attributes', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.description :*)' + 'foo.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -341,7 +341,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: null, isSavedObjectAttr: true, - key: 'foo.updatedAt', + key: 'foo.updated_at', type: 'foo', }, { @@ -387,7 +387,7 @@ describe('Filter Utils', () => { test('Return Error if filter is not using an allowed type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'bar.updatedAt: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'bar.updated_at: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.title: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -398,7 +398,7 @@ describe('Filter Utils', () => { astPath: 'arguments.0', error: 'This type bar is not allowed', isSavedObjectAttr: true, - key: 'bar.updatedAt', + key: 'bar.updated_at', type: 'bar', }, { @@ -442,7 +442,7 @@ describe('Filter Utils', () => { test('Return Error if filter is using an non-existing key in the index patterns of the saved object type', () => { const validationObject = validateFilterKueryNode({ astFilter: esKuery.fromKueryExpression( - 'foo.updatedAt33: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.header: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' + 'foo.updated_at33: 5678654567 and foo.attributes.bytes > 1000 and foo.attributes.bytes < 8000 and foo.attributes.header: "best" and (foo.attributes.description: t* or foo.attributes.description :*)' ), types: ['foo'], indexMapping: mockMappings, @@ -451,9 +451,9 @@ describe('Filter Utils', () => { expect(validationObject).toEqual([ { astPath: 'arguments.0', - error: "This key 'foo.updatedAt33' does NOT exist in foo saved object index patterns", + error: "This key 'foo.updated_at33' does NOT exist in foo saved object index patterns", isSavedObjectAttr: false, - key: 'foo.updatedAt33', + key: 'foo.updated_at33', type: 'foo', }, { @@ -519,6 +519,33 @@ describe('Filter Utils', () => { }, ]); }); + + test('Validate multiple items nested filter query through KueryNode', () => { + const validationObject = validateFilterKueryNode({ + astFilter: esKuery.fromKueryExpression( + 'alert.attributes.actions:{ actionTypeId: ".server-log" AND actionRef: "foo" }' + ), + types: ['alert'], + indexMapping: mockMappings, + }); + + expect(validationObject).toEqual([ + { + astPath: 'arguments.1.arguments.0', + error: null, + isSavedObjectAttr: false, + key: 'alert.attributes.actions.actionTypeId', + type: 'alert', + }, + { + astPath: 'arguments.1.arguments.1', + error: null, + isSavedObjectAttr: false, + key: 'alert.attributes.actions.actionRef', + type: 'alert', + }, + ]); + }); }); describe('#hasFilterKeyError', () => { diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index 688b7ad96e8ed..b3bcef9a62e13 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -109,7 +109,15 @@ export const validateFilterKueryNode = ({ return astFilter.arguments.reduce((kueryNode: string[], ast: KueryNode, index: number) => { if (hasNestedKey && ast.type === 'literal' && ast.value != null) { localNestedKeys = ast.value; + } else if (ast.type === 'literal' && ast.value && typeof ast.value === 'string') { + const key = ast.value.replace('.attributes', ''); + const mappingKey = 'properties.' + key.split('.').join('.properties.'); + const field = get(indexMapping, mappingKey); + if (field != null && field.type === 'nested') { + localNestedKeys = ast.value; + } } + if (ast.arguments) { const myPath = `${path}.${index}`; return [ @@ -121,7 +129,7 @@ export const validateFilterKueryNode = ({ storeValue: ast.type === 'function' && astFunctionType.includes(ast.function), path: `${myPath}.arguments`, hasNestedKey: ast.type === 'function' && ast.function === 'nested', - nestedKeys: localNestedKeys, + nestedKeys: localNestedKeys || nestedKeys, }), ]; } @@ -226,7 +234,7 @@ export const fieldDefined = (indexMappings: IndexMapping, key: string): boolean return true; } - // If the path is for a flattned type field, we'll assume the mappings are defined. + // If the path is for a flattened type field, we'll assume the mappings are defined. const keys = key.split('.'); for (let i = 0; i < keys.length; i++) { const path = `properties.${keys.slice(0, i + 1).join('.properties.')}`; diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 7c719ac56a835..c0e2cdc333363 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -66,6 +66,7 @@ import { import { LegacyUrlAlias, LEGACY_URL_ALIAS_TYPE } from '../../object_types'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { validateConvertFilterToKueryNode } from './filter_utils'; +import { validateAndConvertAggregations } from './aggregations'; import { ALL_NAMESPACES_STRING, FIND_DEFAULT_PAGE, @@ -748,7 +749,9 @@ export class SavedObjectsRepository { * @property {string} [options.preference] * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ - async find(options: SavedObjectsFindOptions): Promise> { + async find( + options: SavedObjectsFindOptions + ): Promise> { const { search, defaultSearchOperator = 'OR', @@ -768,6 +771,7 @@ export class SavedObjectsRepository { typeToNamespacesMap, filter, preference, + aggs, } = options; if (!type && !typeToNamespacesMap) { @@ -799,7 +803,7 @@ export class SavedObjectsRepository { : Array.from(typeToNamespacesMap!.keys()); const allowedTypes = types.filter((t) => this._allowedTypes.includes(t)); if (allowedTypes.length === 0) { - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } if (searchFields && !Array.isArray(searchFields)) { @@ -811,16 +815,24 @@ export class SavedObjectsRepository { } let kueryNode; - - try { - if (filter) { + if (filter) { + try { kueryNode = validateConvertFilterToKueryNode(allowedTypes, filter, this._mappings); + } catch (e) { + if (e.name === 'KQLSyntaxError') { + throw SavedObjectsErrorHelpers.createBadRequestError(`KQLSyntaxError: ${e.message}`); + } else { + throw e; + } } - } catch (e) { - if (e.name === 'KQLSyntaxError') { - throw SavedObjectsErrorHelpers.createBadRequestError('KQLSyntaxError: ' + e.message); - } else { - throw e; + } + + let aggsObject; + if (aggs) { + try { + aggsObject = validateAndConvertAggregations(allowedTypes, aggs, this._mappings); + } catch (e) { + throw SavedObjectsErrorHelpers.createBadRequestError(`Invalid aggregation: ${e.message}`); } } @@ -838,6 +850,7 @@ export class SavedObjectsRepository { seq_no_primary_term: true, from: perPage * (page - 1), _source: includedFields(type, fields), + ...(aggsObject ? { aggs: aggsObject } : {}), ...getSearchDsl(this._mappings, this._registry, { search, defaultSearchOperator, @@ -872,6 +885,7 @@ export class SavedObjectsRepository { } return { + ...(body.aggregations ? { aggregations: (body.aggregations as unknown) as A } : {}), page, per_page: perPage, total: body.hits.total, @@ -885,7 +899,7 @@ export class SavedObjectsRepository { }) ), pit_id: body.pit_id, - } as SavedObjectsFindResponse; + } as SavedObjectsFindResponse; } /** diff --git a/src/core/server/saved_objects/service/lib/utils.ts b/src/core/server/saved_objects/service/lib/utils.ts index ebad13e5edc25..494ac6ce9fad5 100644 --- a/src/core/server/saved_objects/service/lib/utils.ts +++ b/src/core/server/saved_objects/service/lib/utils.ts @@ -51,10 +51,10 @@ export class SavedObjectsUtils { /** * Creates an empty response for a find operation. This is only intended to be used by saved objects client wrappers. */ - public static createEmptyFindResponse = ({ + public static createEmptyFindResponse = ({ page = FIND_DEFAULT_PAGE, perPage = FIND_DEFAULT_PER_PAGE, - }: SavedObjectsFindOptions): SavedObjectsFindResponse => ({ + }: SavedObjectsFindOptions): SavedObjectsFindResponse => ({ page, per_page: perPage, total: 0, diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 9a0ccb88d3555..12451ace02836 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -173,7 +173,8 @@ export interface SavedObjectsFindResult extends SavedObject { * * @public */ -export interface SavedObjectsFindResponse { +export interface SavedObjectsFindResponse { + aggregations?: A; saved_objects: Array>; total: number; per_page: number; @@ -463,7 +464,9 @@ export class SavedObjectsClient { * * @param options */ - async find(options: SavedObjectsFindOptions): Promise> { + async find( + options: SavedObjectsFindOptions + ): Promise> { return await this._repository.find(options); } diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index ecda120e025d8..d3bfdcc6923dc 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -116,6 +116,28 @@ export interface SavedObjectsFindOptions { */ defaultSearchOperator?: 'AND' | 'OR'; filter?: string | KueryNode; + /** + * A record of aggregations to perform. + * The API currently only supports a limited set of metrics and bucket aggregation types. + * Additional aggregation types can be contributed to Core. + * + * @example + * Aggregating on SO attribute field + * ```ts + * const aggs = { latest_version: { max: { field: 'dashboard.attributes.version' } } }; + * return client.find({ type: 'dashboard', aggs }) + * ``` + * + * @example + * Aggregating on SO root field + * ```ts + * const aggs = { latest_update: { max: { field: 'dashboard.updated_at' } } }; + * return client.find({ type: 'dashboard', aggs }) + * ``` + * + * @alpha + */ + aggs?: Record; namespaces?: string[]; /** * This map defines each type to search for, and the namespace(s) to search for the type in; this is only intended to be used by a saved diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 05af684053f39..e8f9dab435754 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2244,7 +2244,7 @@ export class SavedObjectsClient { static errors: typeof SavedObjectsErrorHelpers; // (undocumented) errors: typeof SavedObjectsErrorHelpers; - find(options: SavedObjectsFindOptions): Promise>; + find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; openPointInTimeForType(type: string | string[], options?: SavedObjectsOpenPointInTimeOptions): Promise; removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise; @@ -2501,6 +2501,8 @@ export type SavedObjectsFieldMapping = SavedObjectsCoreFieldMapping | SavedObjec // @public (undocumented) export interface SavedObjectsFindOptions { + // @alpha + aggs?: Record; defaultSearchOperator?: 'AND' | 'OR'; fields?: string[]; // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts @@ -2539,7 +2541,9 @@ export interface SavedObjectsFindOptionsReference { } // @public -export interface SavedObjectsFindResponse { +export interface SavedObjectsFindResponse { + // (undocumented) + aggregations?: A; // (undocumented) page: number; // (undocumented) @@ -2849,7 +2853,7 @@ export class SavedObjectsRepository { deleteByNamespace(namespace: string, options?: SavedObjectsDeleteByNamespaceOptions): Promise; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise; // (undocumented) - find(options: SavedObjectsFindOptions): Promise>; + find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>; openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise; @@ -2970,7 +2974,7 @@ export interface SavedObjectsUpdateResponse extends Omit({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; + static createEmptyFindResponse: ({ page, perPage, }: SavedObjectsFindOptions) => SavedObjectsFindResponse; static generateId(): string; static isRandomId(id: string | undefined): boolean; static namespaceIdToString: (namespace?: string | undefined) => string; diff --git a/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts b/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts index d639b053565d1..01d89c5731158 100644 --- a/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts +++ b/src/plugins/telemetry_collection_manager/server/telemetry_saved_objects_client.ts @@ -17,7 +17,9 @@ export class TelemetrySavedObjectsClient extends SavedObjectsClient { * Find the SavedObjects matching the search query in all the Spaces by default * @param options */ - async find(options: SavedObjectsFindOptions): Promise> { + async find( + options: SavedObjectsFindOptions + ): Promise> { return super.find({ namespaces: ['*'], ...options }); } } diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index 28c38ca9e0ded..a01562861e606 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -293,6 +293,75 @@ export default function ({ getService }: FtrProviderContext) { })); }); + describe('using aggregations', () => { + it('should return 200 with valid response for a valid aggregation', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'visualization.attributes.version' } }, + }) + )}` + ) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + aggregations: { + type_count: { + value: 1, + }, + }, + page: 1, + per_page: 0, + saved_objects: [], + total: 1, + }); + })); + + it('should return a 400 when referencing an invalid SO attribute', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { max: { field: 'dashboard.attributes.version' } }, + }) + )}` + ) + .expect(400) + .then((resp) => { + expect(resp.body).to.eql({ + error: 'Bad Request', + message: + 'Invalid aggregation: [type_count.max.field] Invalid attribute path: dashboard.attributes.version: Bad Request', + statusCode: 400, + }); + })); + + it('should return a 400 when using a forbidden aggregation option', async () => + await supertest + .get( + `/api/saved_objects/_find?type=visualization&per_page=0&aggs=${encodeURIComponent( + JSON.stringify({ + type_count: { + max: { + field: 'visualization.attributes.version', + script: 'Bad script is bad', + }, + }, + }) + )}` + ) + .expect(400) + .then((resp) => { + expect(resp.body).to.eql({ + error: 'Bad Request', + message: + 'Invalid aggregation: [type_count.max.script]: definition for this key is missing: Bad Request', + statusCode: 400, + }); + })); + }); + describe('`has_reference` and `has_reference_operator` parameters', () => { before(() => esArchiver.load('saved_objects/references')); after(() => esArchiver.unload('saved_objects/references')); diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index 88a89af6be3d0..9b699d6ce007c 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -162,9 +162,9 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.delete(type, id, options); } - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { return await this.handleEncryptedAttributesInBulkResponse( - await this.options.baseClient.find(options), + await this.options.baseClient.find(options), undefined ); } diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index 8378cc4d848cf..d876175a05fe8 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -213,7 +213,7 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return await this.baseClient.delete(type, id, options); } - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { if ( this.getSpacesService() == null && Array.isArray(options.namespaces) && @@ -245,7 +245,7 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra error: new Error(status), }) ); - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } const typeToNamespacesMap = Array.from(typeMap).reduce>( @@ -254,7 +254,7 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra new Map() ); - const response = await this.baseClient.find({ + const response = await this.baseClient.find({ ...options, typeToNamespacesMap: undefined, // if the user is fully authorized, use `undefined` as the typeToNamespacesMap to prevent privilege escalation ...(status === 'partially_authorized' && { typeToNamespacesMap, type: '', namespaces: [] }), // the repository requires that `type` and `namespaces` must be empty if `typeToNamespacesMap` is defined diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index c544e2f46f058..4254615ac7d5f 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -171,7 +171,7 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * @property {object} [options.hasReference] - { type, id } * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ - public async find(options: SavedObjectsFindOptions) { + public async find(options: SavedObjectsFindOptions) { throwErrorIfNamespaceSpecified(options); let namespaces = options.namespaces; @@ -187,12 +187,12 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { } if (namespaces.length === 0) { // return empty response, since the user is unauthorized in this space (or these spaces), but we don't return forbidden errors for `find` operations - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } } catch (err) { if (Boom.isBoom(err) && err.output.payload.statusCode === 403) { // return empty response, since the user is unauthorized in any space, but we don't return forbidden errors for `find` operations - return SavedObjectsUtils.createEmptyFindResponse(options); + return SavedObjectsUtils.createEmptyFindResponse(options); } throw err; } @@ -200,7 +200,7 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { namespaces = [this.spaceId]; } - return await this.client.find({ + return await this.client.find({ ...options, type: (options.type ? coerceToArray(options.type) : this.types).filter( (type) => type !== 'space' From c187270b5e2d90c306a55694c53614e149422d35 Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Fri, 16 Apr 2021 19:59:23 +0300 Subject: [PATCH 11/25] [Search Sessions] Client side search cache (#92439) * dev docs * sessions tutorial * title * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Update dev_docs/tutorials/data/search.mdx Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> * Code review * client cache * mock utils * improve code * Use cacheOnClient in Lens * mock * docs and types * unit tests! * Search response cache + tests * remove cacheOnClient evict cache on error * test ts * shouldCacheOnClient + improve tests * remove unused * clear subs * dont unsubscribe on setItem * caching mess * t * fix jest * add size to bfetch response @ppisljar use it to reduce the # of stringify in response cache * ts * ts * docs * simplify abort controller logic and extract it into a class * docs * delete unused tests * use addAbortSignal * code review * Use shareReplay, fix tests * code review * bfetch test * code review * Leave the bfetch changes out * docs + isRestore * make sure to clean up properly * Make sure that aborting in cache works correctly Clearer restructuring of code * fix test * import * code review round 1 * ts * Added functional test for search request caching * test * skip before codefreeze Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...earchinterceptor.getserializableoptions.md | 22 + ...n-plugins-data-public.searchinterceptor.md | 1 + examples/search_examples/common/index.ts | 1 + .../search_examples/public/search/app.tsx | 67 ++- .../search_examples/server/my_strategy.ts | 1 + .../data/common/search/tabify/tabify.ts | 2 +- src/plugins/data/public/public.api.md | 2 + .../data/public/search/search_interceptor.ts | 28 +- .../public/search/session/session_service.ts | 2 +- .../search/search_abort_controller.test.ts | 22 +- .../public/search/search_abort_controller.ts | 7 +- .../public/search/search_interceptor.test.ts | 557 +++++++++++++++++- .../public/search/search_interceptor.ts | 150 ++++- .../search/search_response_cache.test.ts | 318 ++++++++++ .../public/search/search_response_cache.ts | 136 +++++ .../data_enhanced/public/search/utils.ts | 15 + x-pack/plugins/lens/public/app_plugin/app.tsx | 13 +- x-pack/test/examples/search_examples/index.ts | 1 + .../search_examples/search_sessions_cache.ts | 65 ++ 19 files changed, 1350 insertions(+), 60 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md create mode 100644 x-pack/plugins/data_enhanced/public/search/search_response_cache.test.ts create mode 100644 x-pack/plugins/data_enhanced/public/search/search_response_cache.ts create mode 100644 x-pack/plugins/data_enhanced/public/search/utils.ts create mode 100644 x-pack/test/examples/search_examples/search_sessions_cache.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md new file mode 100644 index 0000000000000..984f99004ebe8 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [SearchInterceptor](./kibana-plugin-plugins-data-public.searchinterceptor.md) > [getSerializableOptions](./kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md) + +## SearchInterceptor.getSerializableOptions() method + +Signature: + +```typescript +protected getSerializableOptions(options?: ISearchOptions): Pick; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| options | ISearchOptions | | + +Returns: + +`Pick` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md index 9d18309fc07be..653f052dd5a3a 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchinterceptor.md @@ -26,6 +26,7 @@ export declare class SearchInterceptor | Method | Modifiers | Description | | --- | --- | --- | +| [getSerializableOptions(options)](./kibana-plugin-plugins-data-public.searchinterceptor.getserializableoptions.md) | | | | [getTimeoutMode()](./kibana-plugin-plugins-data-public.searchinterceptor.gettimeoutmode.md) | | | | [handleSearchError(e, options, isTimeout)](./kibana-plugin-plugins-data-public.searchinterceptor.handlesearcherror.md) | | | | [search(request, options)](./kibana-plugin-plugins-data-public.searchinterceptor.search.md) | | Searches using the given search method. Overrides the AbortSignal with one that will abort either when the request times out, or when the original AbortSignal is aborted. Updates pendingCount$ when the request is started/finalized. | diff --git a/examples/search_examples/common/index.ts b/examples/search_examples/common/index.ts index dd953b1ec8982..cc47c0f575973 100644 --- a/examples/search_examples/common/index.ts +++ b/examples/search_examples/common/index.ts @@ -16,6 +16,7 @@ export interface IMyStrategyRequest extends IEsSearchRequest { } export interface IMyStrategyResponse extends IEsSearchResponse { cool: string; + executed_at: number; } export const SERVER_SEARCH_ROUTE_PATH = '/api/examples/search'; diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index 3bac445581ae7..8f31d242faf5e 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -111,7 +111,7 @@ export const SearchExamplesApp = ({ setSelectedNumericField(fields?.length ? getNumeric(fields)[0] : null); }, [fields]); - const doAsyncSearch = async (strategy?: string) => { + const doAsyncSearch = async (strategy?: string, sessionId?: string) => { if (!indexPattern || !selectedNumericField) return; // Construct the query portion of the search request @@ -138,6 +138,7 @@ export const SearchExamplesApp = ({ const searchSubscription$ = data.search .search(req, { strategy, + sessionId, }) .subscribe({ next: (res) => { @@ -148,19 +149,30 @@ export const SearchExamplesApp = ({ ? // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response res.rawResponse.aggregations[1].value : undefined; + const isCool = (res as IMyStrategyResponse).cool; + const executedAt = (res as IMyStrategyResponse).executed_at; const message = ( Searched {res.rawResponse.hits.total} documents.
The average of {selectedNumericField!.name} is{' '} {avgResult ? Math.floor(avgResult) : 0}.
- Is it Cool? {String((res as IMyStrategyResponse).cool)} + {isCool ? `Is it Cool? ${isCool}` : undefined} +
+ + {executedAt ? `Executed at? ${executedAt}` : undefined} + ); - notifications.toasts.addSuccess({ - title: 'Query result', - text: mountReactNode(message), - }); + notifications.toasts.addSuccess( + { + title: 'Query result', + text: mountReactNode(message), + }, + { + toastLifeTimeMs: 300000, + } + ); searchSubscription$.unsubscribe(); } else if (isErrorResponse(res)) { // TODO: Make response error status clearer @@ -227,6 +239,10 @@ export const SearchExamplesApp = ({ doAsyncSearch('myStrategy'); }; + const onClientSideSessionCacheClickHandler = () => { + doAsyncSearch('myStrategy', data.search.session.getSessionId()); + }; + const onServerClickHandler = async () => { if (!indexPattern || !selectedNumericField) return; try { @@ -374,6 +390,45 @@ export const SearchExamplesApp = ({ + +

Client side search session caching

+ + + data.search.session.start()} + iconType="alert" + data-test-subj="searchExamplesStartSession" + > + + + data.search.session.clear()} + iconType="alert" + data-test-subj="searchExamplesClearSession" + > + + + + + + +

Using search on the server

 diff --git a/examples/search_examples/server/my_strategy.ts b/examples/search_examples/server/my_strategy.ts index 2cf039e99f6e9..0a64788960091 100644 --- a/examples/search_examples/server/my_strategy.ts +++ b/examples/search_examples/server/my_strategy.ts @@ -20,6 +20,7 @@ export const mySearchStrategyProvider = ( map((esSearchRes) => ({ ...esSearchRes, cool: request.get_cool ? 'YES' : 'NOPE', + executed_at: new Date().getTime(), })) ), cancel: async (id, options, deps) => { diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index 9f096886491ad..4a8972d4384c2 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -139,7 +139,7 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { ...esResponse.aggregations, - doc_count: esResponse.hits.total, + doc_count: esResponse.hits?.total, }; collectBucket(aggConfigs, write, topLevelBucket, '', 1); diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index d99d754a3364d..35f13fc855e99 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -2353,6 +2353,8 @@ export class SearchInterceptor { // (undocumented) protected readonly deps: SearchInterceptorDeps; // (undocumented) + protected getSerializableOptions(options?: ISearchOptions): Pick; + // (undocumented) protected getTimeoutMode(): TimeoutErrorMode; // Warning: (ae-forgotten-export) The symbol "KibanaServerError" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "AbortError" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index 3df2313f83798..e3fb31c9179fd 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -113,20 +113,14 @@ export class SearchInterceptor { } } - /** - * @internal - * @throws `AbortError` | `ErrorLike` - */ - protected runSearch( - request: IKibanaSearchRequest, - options?: ISearchOptions - ): Promise { - const { abortSignal, sessionId, ...requestOptions } = options || {}; + protected getSerializableOptions(options?: ISearchOptions) { + const { sessionId, ...requestOptions } = options || {}; + + const serializableOptions: ISearchOptionsSerializable = {}; const combined = { ...requestOptions, ...this.deps.session.getSearchOptions(sessionId), }; - const serializableOptions: ISearchOptionsSerializable = {}; if (combined.sessionId !== undefined) serializableOptions.sessionId = combined.sessionId; if (combined.isRestore !== undefined) serializableOptions.isRestore = combined.isRestore; @@ -135,10 +129,22 @@ export class SearchInterceptor { if (combined.strategy !== undefined) serializableOptions.strategy = combined.strategy; if (combined.isStored !== undefined) serializableOptions.isStored = combined.isStored; + return serializableOptions; + } + + /** + * @internal + * @throws `AbortError` | `ErrorLike` + */ + protected runSearch( + request: IKibanaSearchRequest, + options?: ISearchOptions + ): Promise { + const { abortSignal } = options || {}; return this.batchedFetch( { request, - options: serializableOptions, + options: this.getSerializableOptions(options), }, abortSignal ); diff --git a/src/plugins/data/public/search/session/session_service.ts b/src/plugins/data/public/search/session/session_service.ts index 381410574ecda..71f51b4bc8d83 100644 --- a/src/plugins/data/public/search/session/session_service.ts +++ b/src/plugins/data/public/search/session/session_service.ts @@ -73,7 +73,7 @@ export interface SearchSessionIndicatorUiConfig { } /** - * Responsible for tracking a current search session. Supports only a single session at a time. + * Responsible for tracking a current search session. Supports a single session at a time. */ export class SessionService { public readonly state$: Observable; diff --git a/x-pack/plugins/data_enhanced/public/search/search_abort_controller.test.ts b/x-pack/plugins/data_enhanced/public/search/search_abort_controller.test.ts index 68282c1e947f7..a52fdef9819b8 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_abort_controller.test.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_abort_controller.test.ts @@ -21,13 +21,15 @@ describe('search abort controller', () => { test('immediately aborts when passed an aborted signal in the constructor', () => { const controller = new AbortController(); controller.abort(); - const sac = new SearchAbortController(controller.signal); + const sac = new SearchAbortController(); + sac.addAbortSignal(controller.signal); expect(sac.getSignal().aborted).toBe(true); }); test('aborts when input signal is aborted', () => { const controller = new AbortController(); - const sac = new SearchAbortController(controller.signal); + const sac = new SearchAbortController(); + sac.addAbortSignal(controller.signal); expect(sac.getSignal().aborted).toBe(false); controller.abort(); expect(sac.getSignal().aborted).toBe(true); @@ -35,7 +37,8 @@ describe('search abort controller', () => { test('aborts when all input signals are aborted', () => { const controller = new AbortController(); - const sac = new SearchAbortController(controller.signal); + const sac = new SearchAbortController(); + sac.addAbortSignal(controller.signal); const controller2 = new AbortController(); sac.addAbortSignal(controller2.signal); @@ -48,7 +51,8 @@ describe('search abort controller', () => { test('aborts explicitly even if all inputs are not aborted', () => { const controller = new AbortController(); - const sac = new SearchAbortController(controller.signal); + const sac = new SearchAbortController(); + sac.addAbortSignal(controller.signal); const controller2 = new AbortController(); sac.addAbortSignal(controller2.signal); @@ -60,7 +64,8 @@ describe('search abort controller', () => { test('doesnt abort, if cleared', () => { const controller = new AbortController(); - const sac = new SearchAbortController(controller.signal); + const sac = new SearchAbortController(); + sac.addAbortSignal(controller.signal); expect(sac.getSignal().aborted).toBe(false); sac.cleanup(); controller.abort(); @@ -77,7 +82,7 @@ describe('search abort controller', () => { }); test('doesnt abort on timeout, if cleared', () => { - const sac = new SearchAbortController(undefined, 100); + const sac = new SearchAbortController(100); expect(sac.getSignal().aborted).toBe(false); sac.cleanup(); timeTravel(100); @@ -85,7 +90,7 @@ describe('search abort controller', () => { }); test('aborts on timeout, even if no signals passed in', () => { - const sac = new SearchAbortController(undefined, 100); + const sac = new SearchAbortController(100); expect(sac.getSignal().aborted).toBe(false); timeTravel(100); expect(sac.getSignal().aborted).toBe(true); @@ -94,7 +99,8 @@ describe('search abort controller', () => { test('aborts on timeout, even if there are unaborted signals', () => { const controller = new AbortController(); - const sac = new SearchAbortController(controller.signal, 100); + const sac = new SearchAbortController(100); + sac.addAbortSignal(controller.signal); expect(sac.getSignal().aborted).toBe(false); timeTravel(100); diff --git a/x-pack/plugins/data_enhanced/public/search/search_abort_controller.ts b/x-pack/plugins/data_enhanced/public/search/search_abort_controller.ts index 4482a7771dc28..7bc74b56a3903 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_abort_controller.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_abort_controller.ts @@ -18,11 +18,7 @@ export class SearchAbortController { private destroyed = false; private reason?: AbortReason; - constructor(abortSignal?: AbortSignal, timeout?: number) { - if (abortSignal) { - this.addAbortSignal(abortSignal); - } - + constructor(timeout?: number) { if (timeout) { this.timeoutSub = timer(timeout).subscribe(() => { this.reason = AbortReason.Timeout; @@ -41,6 +37,7 @@ export class SearchAbortController { }; public cleanup() { + if (this.destroyed) return; this.destroyed = true; this.timeoutSub?.unsubscribe(); this.inputAbortSignals.forEach((abortSignal) => { diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts index 02671974e5053..0e511c545f3e2 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.test.ts @@ -23,9 +23,12 @@ import { bfetchPluginMock } from '../../../../../src/plugins/bfetch/public/mocks import { BehaviorSubject } from 'rxjs'; import * as xpackResourceNotFoundException from '../../common/search/test_data/search_phase_execution_exception.json'; -const timeTravel = (msToRun = 0) => { +const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); + +const timeTravel = async (msToRun = 0) => { + await flushPromises(); jest.advanceTimersByTime(msToRun); - return new Promise((resolve) => setImmediate(resolve)); + return flushPromises(); }; const next = jest.fn(); @@ -39,10 +42,20 @@ let fetchMock: jest.Mock; jest.useFakeTimers(); +jest.mock('./utils', () => ({ + createRequestHash: jest.fn().mockImplementation((input) => { + return Promise.resolve(JSON.stringify(input)); + }), +})); + function mockFetchImplementation(responses: any[]) { let i = 0; - fetchMock.mockImplementation(() => { + fetchMock.mockImplementation((r) => { + if (!r.request.id) i = 0; const { time = 0, value = {}, isError = false } = responses[i++]; + value.meta = { + size: 10, + }; return new Promise((resolve, reject) => setTimeout(() => { return (isError ? reject : resolve)(value); @@ -452,7 +465,7 @@ describe('EnhancedSearchInterceptor', () => { }); }); - describe('session', () => { + describe('session tracking', () => { beforeEach(() => { const responses = [ { @@ -559,4 +572,540 @@ describe('EnhancedSearchInterceptor', () => { expect(sessionService.trackSearch).toBeCalledTimes(0); }); }); + + describe('session client caching', () => { + const sessionId = 'sessionId'; + const basicReq = { + params: { + test: 1, + }, + }; + + const basicCompleteResponse = [ + { + time: 10, + value: { + isPartial: false, + isRunning: false, + id: 1, + rawResponse: { + took: 1, + }, + }, + }, + ]; + + const partialCompleteResponse = [ + { + time: 10, + value: { + isPartial: true, + isRunning: true, + id: 1, + rawResponse: { + took: 1, + }, + }, + }, + { + time: 20, + value: { + isPartial: false, + isRunning: false, + id: 1, + rawResponse: { + took: 1, + }, + }, + }, + ]; + + beforeEach(() => { + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + }); + + test('should be disabled if there is no session', async () => { + mockFetchImplementation(basicCompleteResponse); + + searchInterceptor.search(basicReq, {}).subscribe({ next, error, complete }); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(basicReq, {}).subscribe({ next, error, complete }); + expect(fetchMock).toBeCalledTimes(2); + }); + + test('should fetch different requests in a single session', async () => { + mockFetchImplementation(basicCompleteResponse); + + const req2 = { + params: { + test: 2, + }, + }; + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(req2, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(2); + }); + + test('should fetch the same request for two different sessions', async () => { + mockFetchImplementation(basicCompleteResponse); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor + .search(basicReq, { sessionId: 'anotherSession' }) + .subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(2); + }); + + test('should track searches that come from cache', async () => { + mockFetchImplementation(partialCompleteResponse); + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + + const untrack = jest.fn(); + sessionService.trackSearch.mockImplementation(() => untrack); + + const req = { + params: { + test: 200, + }, + }; + + const response = searchInterceptor.search(req, { pollInterval: 1, sessionId }); + const response2 = searchInterceptor.search(req, { pollInterval: 1, sessionId }); + response.subscribe({ next, error, complete }); + response2.subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + expect(sessionService.trackSearch).toBeCalledTimes(2); + expect(untrack).not.toBeCalled(); + await timeTravel(300); + // Should be called only 2 times (once per partial response) + expect(fetchMock).toBeCalledTimes(2); + expect(sessionService.trackSearch).toBeCalledTimes(2); + expect(untrack).toBeCalledTimes(2); + + expect(next).toBeCalledTimes(4); + expect(error).toBeCalledTimes(0); + expect(complete).toBeCalledTimes(2); + }); + + test('should cache partial responses', async () => { + const responses = [ + { + time: 10, + value: { + isPartial: true, + isRunning: true, + id: 1, + }, + }, + ]; + + mockFetchImplementation(responses); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + }); + + test('should not cache error responses', async () => { + const responses = [ + { + time: 10, + value: { + isPartial: true, + isRunning: false, + id: 1, + }, + }, + ]; + + mockFetchImplementation(responses); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(2); + }); + + test('should deliver error to all replays', async () => { + const responses = [ + { + time: 10, + value: { + isPartial: true, + isRunning: false, + id: 1, + }, + }, + ]; + + mockFetchImplementation(responses); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + expect(error).toBeCalledTimes(2); + expect(error.mock.calls[0][0].message).toEqual('Received partial response'); + expect(error.mock.calls[1][0].message).toEqual('Received partial response'); + }); + + test('should ignore anything outside params when hashing', async () => { + mockFetchImplementation(basicCompleteResponse); + + const req = { + something: 123, + params: { + test: 1, + }, + }; + + const req2 = { + something: 321, + params: { + test: 1, + }, + }; + + searchInterceptor.search(req, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(req2, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + }); + + test('should ignore preference when hashing', async () => { + mockFetchImplementation(basicCompleteResponse); + + const req = { + params: { + test: 1, + preference: 123, + }, + }; + + const req2 = { + params: { + test: 1, + preference: 321, + }, + }; + + searchInterceptor.search(req, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(req2, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + }); + + test('should return from cache for identical requests in the same session', async () => { + mockFetchImplementation(basicCompleteResponse); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + }); + + test('aborting a search that didnt get any response should retrigger search', async () => { + mockFetchImplementation(basicCompleteResponse); + + const abortController = new AbortController(); + + // Start a search request + searchInterceptor + .search(basicReq, { sessionId, abortSignal: abortController.signal }) + .subscribe({ next, error, complete }); + + // Abort the search request before it started + abortController.abort(); + + // Time travel to make sure nothing appens + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(0); + expect(next).toBeCalledTimes(0); + expect(error).toBeCalledTimes(1); + expect(complete).toBeCalledTimes(0); + + const error2 = jest.fn(); + const next2 = jest.fn(); + const complete2 = jest.fn(); + + // Search for the same thing again + searchInterceptor + .search(basicReq, { sessionId }) + .subscribe({ next: next2, error: error2, complete: complete2 }); + + // Should search again + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + expect(next2).toBeCalledTimes(1); + expect(error2).toBeCalledTimes(0); + expect(complete2).toBeCalledTimes(1); + }); + + test('aborting a running first search shouldnt clear cache', async () => { + mockFetchImplementation(partialCompleteResponse); + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + + const untrack = jest.fn(); + sessionService.trackSearch.mockImplementation(() => untrack); + + const req = { + params: { + test: 200, + }, + }; + + const abortController = new AbortController(); + + const response = searchInterceptor.search(req, { + pollInterval: 1, + sessionId, + abortSignal: abortController.signal, + }); + response.subscribe({ next, error, complete }); + await timeTravel(10); + + expect(fetchMock).toBeCalledTimes(1); + expect(next).toBeCalledTimes(1); + expect(error).toBeCalledTimes(0); + expect(complete).toBeCalledTimes(0); + expect(sessionService.trackSearch).toBeCalledTimes(1); + expect(untrack).not.toBeCalled(); + + const next2 = jest.fn(); + const error2 = jest.fn(); + const complete2 = jest.fn(); + const response2 = searchInterceptor.search(req, { pollInterval: 1, sessionId }); + response2.subscribe({ next: next2, error: error2, complete: complete2 }); + await timeTravel(0); + + abortController.abort(); + + await timeTravel(300); + // Both searches should be tracked and untracked + expect(sessionService.trackSearch).toBeCalledTimes(2); + expect(untrack).toBeCalledTimes(2); + + // First search should error + expect(next).toBeCalledTimes(1); + expect(error).toBeCalledTimes(1); + expect(complete).toBeCalledTimes(0); + + // Second search should complete + expect(next2).toBeCalledTimes(2); + expect(error2).toBeCalledTimes(0); + expect(complete2).toBeCalledTimes(1); + + // Should be called only 2 times (once per partial response) + expect(fetchMock).toBeCalledTimes(2); + }); + + test('aborting a running second search shouldnt clear cache', async () => { + mockFetchImplementation(partialCompleteResponse); + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + + const untrack = jest.fn(); + sessionService.trackSearch.mockImplementation(() => untrack); + + const req = { + params: { + test: 200, + }, + }; + + const abortController = new AbortController(); + + const response = searchInterceptor.search(req, { pollInterval: 1, sessionId }); + response.subscribe({ next, error, complete }); + await timeTravel(10); + + expect(fetchMock).toBeCalledTimes(1); + expect(next).toBeCalledTimes(1); + expect(error).toBeCalledTimes(0); + expect(complete).toBeCalledTimes(0); + expect(sessionService.trackSearch).toBeCalledTimes(1); + expect(untrack).not.toBeCalled(); + + const next2 = jest.fn(); + const error2 = jest.fn(); + const complete2 = jest.fn(); + const response2 = searchInterceptor.search(req, { + pollInterval: 0, + sessionId, + abortSignal: abortController.signal, + }); + response2.subscribe({ next: next2, error: error2, complete: complete2 }); + await timeTravel(0); + + abortController.abort(); + + await timeTravel(300); + expect(sessionService.trackSearch).toBeCalledTimes(2); + expect(untrack).toBeCalledTimes(2); + + expect(next).toBeCalledTimes(2); + expect(error).toBeCalledTimes(0); + expect(complete).toBeCalledTimes(1); + + expect(next2).toBeCalledTimes(1); + expect(error2).toBeCalledTimes(1); + expect(complete2).toBeCalledTimes(0); + + // Should be called only 2 times (once per partial response) + expect(fetchMock).toBeCalledTimes(2); + }); + + test('aborting both requests should cancel underlaying search only once', async () => { + mockFetchImplementation(partialCompleteResponse); + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + sessionService.trackSearch.mockImplementation(() => jest.fn()); + + const req = { + params: { + test: 200, + }, + }; + + const abortController = new AbortController(); + + const response = searchInterceptor.search(req, { + pollInterval: 1, + sessionId, + abortSignal: abortController.signal, + }); + response.subscribe({ next, error, complete }); + + const response2 = searchInterceptor.search(req, { + pollInterval: 1, + sessionId, + abortSignal: abortController.signal, + }); + response2.subscribe({ next, error, complete }); + await timeTravel(10); + + abortController.abort(); + + await timeTravel(300); + + expect(mockCoreSetup.http.delete).toHaveBeenCalledTimes(1); + }); + + test('aborting both searches should stop searching and clear cache', async () => { + mockFetchImplementation(partialCompleteResponse); + sessionService.isCurrentSession.mockImplementation((_sessionId) => _sessionId === sessionId); + sessionService.getSessionId.mockImplementation(() => sessionId); + + const untrack = jest.fn(); + sessionService.trackSearch.mockImplementation(() => untrack); + + const req = { + params: { + test: 200, + }, + }; + + const abortController = new AbortController(); + + const response = searchInterceptor.search(req, { + pollInterval: 1, + sessionId, + abortSignal: abortController.signal, + }); + response.subscribe({ next, error, complete }); + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + const response2 = searchInterceptor.search(req, { + pollInterval: 1, + sessionId, + abortSignal: abortController.signal, + }); + response2.subscribe({ next, error, complete }); + await timeTravel(0); + expect(fetchMock).toBeCalledTimes(1); + + abortController.abort(); + + await timeTravel(300); + + expect(next).toBeCalledTimes(2); + expect(error).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(0); + expect(error.mock.calls[0][0]).toBeInstanceOf(AbortError); + expect(error.mock.calls[1][0]).toBeInstanceOf(AbortError); + + // Should be called only 1 times (one partial response) + expect(fetchMock).toBeCalledTimes(1); + + // Clear mock and research + fetchMock.mockReset(); + mockFetchImplementation(partialCompleteResponse); + // Run the search again to see that we don't hit the cache + const response3 = searchInterceptor.search(req, { pollInterval: 1, sessionId }); + response3.subscribe({ next, error, complete }); + + await timeTravel(10); + await timeTravel(10); + await timeTravel(300); + + // Should be called 2 times (two partial response) + expect(fetchMock).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(1); + }); + + test('aborting a completed search shouldnt effect cache', async () => { + mockFetchImplementation(basicCompleteResponse); + + const abortController = new AbortController(); + + // Start a search request + searchInterceptor + .search(basicReq, { sessionId, abortSignal: abortController.signal }) + .subscribe({ next, error, complete }); + + // Get a final response + await timeTravel(10); + expect(fetchMock).toBeCalledTimes(1); + + // Abort the search request + abortController.abort(); + + // Search for the same thing again + searchInterceptor.search(basicReq, { sessionId }).subscribe({ next, error, complete }); + + // Get the response from cache + expect(fetchMock).toBeCalledTimes(1); + }); + }); }); diff --git a/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts b/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts index b9d8553d3dc5a..3e7564933a0c6 100644 --- a/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts +++ b/x-pack/plugins/data_enhanced/public/search/search_interceptor.ts @@ -6,8 +6,19 @@ */ import { once } from 'lodash'; -import { throwError, Subscription } from 'rxjs'; -import { tap, finalize, catchError, filter, take, skip } from 'rxjs/operators'; +import { throwError, Subscription, from, of, fromEvent, EMPTY } from 'rxjs'; +import { + tap, + finalize, + catchError, + filter, + take, + skip, + switchMap, + shareReplay, + map, + takeUntil, +} from 'rxjs/operators'; import { TimeoutErrorMode, SearchInterceptor, @@ -16,12 +27,21 @@ import { IKibanaSearchRequest, SearchSessionState, } from '../../../../../src/plugins/data/public'; +import { AbortError } from '../../../../../src/plugins/kibana_utils/public'; import { ENHANCED_ES_SEARCH_STRATEGY, IAsyncSearchOptions, pollSearch } from '../../common'; +import { SearchResponseCache } from './search_response_cache'; +import { createRequestHash } from './utils'; import { SearchAbortController } from './search_abort_controller'; +const MAX_CACHE_ITEMS = 50; +const MAX_CACHE_SIZE_MB = 10; export class EnhancedSearchInterceptor extends SearchInterceptor { private uiSettingsSub: Subscription; private searchTimeout: number; + private readonly responseCache: SearchResponseCache = new SearchResponseCache( + MAX_CACHE_ITEMS, + MAX_CACHE_SIZE_MB + ); /** * @internal @@ -38,6 +58,7 @@ export class EnhancedSearchInterceptor extends SearchInterceptor { } public stop() { + this.responseCache.clear(); this.uiSettingsSub.unsubscribe(); } @@ -47,19 +68,31 @@ export class EnhancedSearchInterceptor extends SearchInterceptor { : TimeoutErrorMode.CONTACT; } - public search({ id, ...request }: IKibanaSearchRequest, options: IAsyncSearchOptions = {}) { - const searchOptions = { - strategy: ENHANCED_ES_SEARCH_STRATEGY, - ...options, + private createRequestHash$(request: IKibanaSearchRequest, options: IAsyncSearchOptions) { + const { sessionId, isRestore } = options; + // Preference is used to ensure all queries go to the same set of shards and it doesn't need to be hashed + // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-shard-routing.html#shard-and-node-preference + const { preference, ...params } = request.params || {}; + const hashOptions = { + ...params, + sessionId, + isRestore, }; - const { sessionId, strategy, abortSignal } = searchOptions; - const search = () => this.runSearch({ id, ...request }, searchOptions); - const searchAbortController = new SearchAbortController(abortSignal, this.searchTimeout); - this.pendingCount$.next(this.pendingCount$.getValue() + 1); - const untrackSearch = this.deps.session.isCurrentSession(options.sessionId) - ? this.deps.session.trackSearch({ abort: () => searchAbortController.abort() }) - : undefined; + return from(sessionId ? createRequestHash(hashOptions) : of(undefined)); + } + + /** + * @internal + * Creates a new pollSearch that share replays its results + */ + private runSearch$( + { id, ...request }: IKibanaSearchRequest, + options: IAsyncSearchOptions, + searchAbortController: SearchAbortController + ) { + const search = () => this.runSearch({ id, ...request }, options); + const { sessionId, strategy } = options; // track if this search's session will be send to background // if yes, then we don't need to cancel this search when it is aborted @@ -91,18 +124,97 @@ export class EnhancedSearchInterceptor extends SearchInterceptor { tap((response) => (id = response.id)), catchError((e: Error) => { cancel(); - return throwError(this.handleSearchError(e, options, searchAbortController.isTimeout())); + return throwError(e); }), finalize(() => { - this.pendingCount$.next(this.pendingCount$.getValue() - 1); searchAbortController.cleanup(); - if (untrackSearch && this.deps.session.isCurrentSession(options.sessionId)) { - // untrack if this search still belongs to current session - untrackSearch(); - } if (savedToBackgroundSub) { savedToBackgroundSub.unsubscribe(); } + }), + // This observable is cached in the responseCache. + // Using shareReplay makes sure that future subscribers will get the final response + + shareReplay(1) + ); + } + + /** + * @internal + * Creates a new search observable and a corresponding search abort controller + * If requestHash is defined, tries to return them first from cache. + */ + private getSearchResponse$( + request: IKibanaSearchRequest, + options: IAsyncSearchOptions, + requestHash?: string + ) { + const cached = requestHash ? this.responseCache.get(requestHash) : undefined; + + const searchAbortController = + cached?.searchAbortController || new SearchAbortController(this.searchTimeout); + + // Create a new abort signal if one was not passed. This fake signal will never be aborted, + // So the underlaying search will not be aborted, even if the other consumers abort. + searchAbortController.addAbortSignal(options.abortSignal ?? new AbortController().signal); + const response$ = cached?.response$ || this.runSearch$(request, options, searchAbortController); + + if (requestHash && !this.responseCache.has(requestHash)) { + this.responseCache.set(requestHash, { + response$, + searchAbortController, + }); + } + + return { + response$, + searchAbortController, + }; + } + + public search({ id, ...request }: IKibanaSearchRequest, options: IAsyncSearchOptions = {}) { + const searchOptions = { + strategy: ENHANCED_ES_SEARCH_STRATEGY, + ...options, + }; + const { sessionId, abortSignal } = searchOptions; + + return this.createRequestHash$(request, searchOptions).pipe( + switchMap((requestHash) => { + const { searchAbortController, response$ } = this.getSearchResponse$( + request, + searchOptions, + requestHash + ); + + this.pendingCount$.next(this.pendingCount$.getValue() + 1); + const untrackSearch = this.deps.session.isCurrentSession(sessionId) + ? this.deps.session.trackSearch({ abort: () => searchAbortController.abort() }) + : undefined; + + // Abort the replay if the abortSignal is aborted. + // The underlaying search will not abort unless searchAbortController fires. + const aborted$ = (abortSignal ? fromEvent(abortSignal, 'abort') : EMPTY).pipe( + map(() => { + throw new AbortError(); + }) + ); + + return response$.pipe( + takeUntil(aborted$), + catchError((e) => { + return throwError( + this.handleSearchError(e, searchOptions, searchAbortController.isTimeout()) + ); + }), + finalize(() => { + this.pendingCount$.next(this.pendingCount$.getValue() - 1); + if (untrackSearch && this.deps.session.isCurrentSession(sessionId)) { + // untrack if this search still belongs to current session + untrackSearch(); + } + }) + ); }) ); } diff --git a/x-pack/plugins/data_enhanced/public/search/search_response_cache.test.ts b/x-pack/plugins/data_enhanced/public/search/search_response_cache.test.ts new file mode 100644 index 0000000000000..e985de5e23f7d --- /dev/null +++ b/x-pack/plugins/data_enhanced/public/search/search_response_cache.test.ts @@ -0,0 +1,318 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { interval, Observable, of, throwError } from 'rxjs'; +import { shareReplay, switchMap, take } from 'rxjs/operators'; +import { IKibanaSearchResponse } from 'src/plugins/data/public'; +import { SearchAbortController } from './search_abort_controller'; +import { SearchResponseCache } from './search_response_cache'; + +describe('SearchResponseCache', () => { + let cache: SearchResponseCache; + let searchAbortController: SearchAbortController; + const r: Array> = [ + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 1, + }, + }, + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 2, + }, + }, + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 3, + }, + }, + { + isPartial: false, + isRunning: false, + rawResponse: { + t: 4, + }, + }, + ]; + + function getSearchObservable$(responses: Array> = r) { + return interval(100).pipe( + take(responses.length), + switchMap((value: number, i: number) => { + if (responses[i].rawResponse.throw === true) { + return throwError('nooo'); + } else { + return of(responses[i]); + } + }), + shareReplay(1) + ); + } + + function wrapWithAbortController(response$: Observable>) { + return { + response$, + searchAbortController, + }; + } + + beforeEach(() => { + cache = new SearchResponseCache(3, 0.1); + searchAbortController = new SearchAbortController(); + }); + + describe('Cache eviction', () => { + test('clear evicts all', () => { + const finalResult = r[r.length - 1]; + cache.set('123', wrapWithAbortController(of(finalResult))); + cache.set('234', wrapWithAbortController(of(finalResult))); + + cache.clear(); + + expect(cache.get('123')).toBeUndefined(); + expect(cache.get('234')).toBeUndefined(); + }); + + test('evicts searches that threw an exception', async () => { + const res$ = getSearchObservable$(); + const err$ = getSearchObservable$([ + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 'a'.repeat(1000), + }, + }, + { + isPartial: true, + isRunning: true, + rawResponse: { + throw: true, + }, + }, + ]); + cache.set('123', wrapWithAbortController(err$)); + cache.set('234', wrapWithAbortController(res$)); + + const errHandler = jest.fn(); + await err$.toPromise().catch(errHandler); + await res$.toPromise().catch(errHandler); + + expect(errHandler).toBeCalledTimes(1); + expect(cache.get('123')).toBeUndefined(); + expect(cache.get('234')).not.toBeUndefined(); + }); + + test('evicts searches that returned an error response', async () => { + const err$ = getSearchObservable$([ + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 1, + }, + }, + { + isPartial: true, + isRunning: false, + rawResponse: { + t: 2, + }, + }, + ]); + cache.set('123', wrapWithAbortController(err$)); + + const errHandler = jest.fn(); + await err$.toPromise().catch(errHandler); + + expect(errHandler).toBeCalledTimes(0); + expect(cache.get('123')).toBeUndefined(); + }); + + test('evicts oldest item if has too many cached items', async () => { + const finalResult = r[r.length - 1]; + cache.set('123', wrapWithAbortController(of(finalResult))); + cache.set('234', wrapWithAbortController(of(finalResult))); + cache.set('345', wrapWithAbortController(of(finalResult))); + cache.set('456', wrapWithAbortController(of(finalResult))); + + expect(cache.get('123')).toBeUndefined(); + expect(cache.get('234')).not.toBeUndefined(); + expect(cache.get('345')).not.toBeUndefined(); + expect(cache.get('456')).not.toBeUndefined(); + }); + + test('evicts oldest item if cache gets bigger than max size', async () => { + const largeResult$ = getSearchObservable$([ + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 'a'.repeat(1000), + }, + }, + { + isPartial: false, + isRunning: false, + rawResponse: { + t: 'a'.repeat(50000), + }, + }, + ]); + + cache.set('123', wrapWithAbortController(largeResult$)); + cache.set('234', wrapWithAbortController(largeResult$)); + cache.set('345', wrapWithAbortController(largeResult$)); + + await largeResult$.toPromise(); + + expect(cache.get('123')).toBeUndefined(); + expect(cache.get('234')).not.toBeUndefined(); + expect(cache.get('345')).not.toBeUndefined(); + }); + + test('evicts from cache any single item that gets bigger than max size', async () => { + const largeResult$ = getSearchObservable$([ + { + isPartial: true, + isRunning: true, + rawResponse: { + t: 'a'.repeat(500), + }, + }, + { + isPartial: false, + isRunning: false, + rawResponse: { + t: 'a'.repeat(500000), + }, + }, + ]); + + cache.set('234', wrapWithAbortController(largeResult$)); + await largeResult$.toPromise(); + expect(cache.get('234')).toBeUndefined(); + }); + + test('get updates the insertion time of an item', async () => { + const finalResult = r[r.length - 1]; + cache.set('123', wrapWithAbortController(of(finalResult))); + cache.set('234', wrapWithAbortController(of(finalResult))); + cache.set('345', wrapWithAbortController(of(finalResult))); + + cache.get('123'); + cache.get('234'); + + cache.set('456', wrapWithAbortController(of(finalResult))); + + expect(cache.get('123')).not.toBeUndefined(); + expect(cache.get('234')).not.toBeUndefined(); + expect(cache.get('345')).toBeUndefined(); + expect(cache.get('456')).not.toBeUndefined(); + }); + }); + + describe('Observable behavior', () => { + test('caches a response and re-emits it', async () => { + const s$ = getSearchObservable$(); + cache.set('123', wrapWithAbortController(s$)); + const finalRes = await cache.get('123')!.response$.toPromise(); + expect(finalRes).toStrictEqual(r[r.length - 1]); + }); + + test('cached$ should emit same as original search$', async () => { + const s$ = getSearchObservable$(); + cache.set('123', wrapWithAbortController(s$)); + + const next = jest.fn(); + const cached$ = cache.get('123'); + + cached$!.response$.subscribe({ + next, + }); + + // wait for original search to complete + await s$!.toPromise(); + + // get final response from cached$ + const finalRes = await cached$!.response$.toPromise(); + expect(finalRes).toStrictEqual(r[r.length - 1]); + expect(next).toHaveBeenCalledTimes(4); + }); + + test('cached$ should emit only current value and keep emitting if subscribed while search$ is running', async () => { + const s$ = getSearchObservable$(); + cache.set('123', wrapWithAbortController(s$)); + + const next = jest.fn(); + let cached$: Observable> | undefined; + s$.subscribe({ + next: (res) => { + if (res.rawResponse.t === 3) { + cached$ = cache.get('123')!.response$; + cached$!.subscribe({ + next, + }); + } + }, + }); + + // wait for original search to complete + await s$!.toPromise(); + + const finalRes = await cached$!.toPromise(); + + expect(finalRes).toStrictEqual(r[r.length - 1]); + expect(next).toHaveBeenCalledTimes(2); + }); + + test('cached$ should emit only last value if subscribed after search$ was complete 1', async () => { + const finalResult = r[r.length - 1]; + const s$ = wrapWithAbortController(of(finalResult)); + cache.set('123', s$); + + // wait for original search to complete + await s$!.response$.toPromise(); + + const next = jest.fn(); + const cached$ = cache.get('123'); + cached$!.response$.subscribe({ + next, + }); + + const finalRes = await cached$!.response$.toPromise(); + + expect(finalRes).toStrictEqual(r[r.length - 1]); + expect(next).toHaveBeenCalledTimes(1); + }); + + test('cached$ should emit only last value if subscribed after search$ was complete', async () => { + const s$ = getSearchObservable$(); + cache.set('123', wrapWithAbortController(s$)); + + // wait for original search to complete + await s$!.toPromise(); + + const next = jest.fn(); + const cached$ = cache.get('123'); + cached$!.response$.subscribe({ + next, + }); + + const finalRes = await cached$!.response$.toPromise(); + + expect(finalRes).toStrictEqual(r[r.length - 1]); + expect(next).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/x-pack/plugins/data_enhanced/public/search/search_response_cache.ts b/x-pack/plugins/data_enhanced/public/search/search_response_cache.ts new file mode 100644 index 0000000000000..1467e5bf234ff --- /dev/null +++ b/x-pack/plugins/data_enhanced/public/search/search_response_cache.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Observable, Subscription } from 'rxjs'; +import { IKibanaSearchResponse, isErrorResponse } from '../../../../../src/plugins/data/public'; +import { SearchAbortController } from './search_abort_controller'; + +interface ResponseCacheItem { + response$: Observable>; + searchAbortController: SearchAbortController; +} + +interface ResponseCacheItemInternal { + response$: Observable>; + searchAbortController: SearchAbortController; + size: number; + subs: Subscription; +} + +export class SearchResponseCache { + private responseCache: Map; + private cacheSize = 0; + + constructor(private maxItems: number, private maxCacheSizeMB: number) { + this.responseCache = new Map(); + } + + private byteToMb(size: number) { + return size / (1024 * 1024); + } + + private deleteItem(key: string, clearSubs = true) { + const item = this.responseCache.get(key); + if (item) { + if (clearSubs) { + item.subs.unsubscribe(); + } + this.cacheSize -= item.size; + this.responseCache.delete(key); + } + } + + private setItem(key: string, item: ResponseCacheItemInternal) { + // The deletion of the key will move it to the end of the Map's entries. + this.deleteItem(key, false); + this.cacheSize += item.size; + this.responseCache.set(key, item); + } + + public clear() { + this.cacheSize = 0; + this.responseCache.forEach((item) => { + item.subs.unsubscribe(); + }); + this.responseCache.clear(); + } + + private shrink() { + while ( + this.responseCache.size > this.maxItems || + this.byteToMb(this.cacheSize) > this.maxCacheSizeMB + ) { + const [key] = [...this.responseCache.keys()]; + this.deleteItem(key); + } + } + + public has(key: string) { + return this.responseCache.has(key); + } + + /** + * + * @param key key to cache + * @param response$ + * @returns A ReplaySubject that mimics the behavior of the original observable + * @throws error if key already exists + */ + public set(key: string, item: ResponseCacheItem) { + if (this.responseCache.has(key)) { + throw new Error('duplicate key'); + } + + const { response$, searchAbortController } = item; + + const cacheItem: ResponseCacheItemInternal = { + response$, + searchAbortController, + subs: new Subscription(), + size: 0, + }; + + this.setItem(key, cacheItem); + + cacheItem.subs.add( + response$.subscribe({ + next: (r) => { + // TODO: avoid stringiying. Get the size some other way! + const newSize = new Blob([JSON.stringify(r)]).size; + if (this.byteToMb(newSize) < this.maxCacheSizeMB && !isErrorResponse(r)) { + this.setItem(key, { + ...cacheItem, + size: newSize, + }); + this.shrink(); + } else { + // Single item is too large to be cached, or an error response returned. + // Evict and ignore. + this.deleteItem(key); + } + }, + error: (e) => { + // Evict item on error + this.deleteItem(key); + }, + }) + ); + this.shrink(); + } + + public get(key: string): ResponseCacheItem | undefined { + const item = this.responseCache.get(key); + if (item) { + // touch the item, and move it to the end of the map's entries + this.setItem(key, item); + return { + response$: item.response$, + searchAbortController: item.searchAbortController, + }; + } + } +} diff --git a/x-pack/plugins/data_enhanced/public/search/utils.ts b/x-pack/plugins/data_enhanced/public/search/utils.ts new file mode 100644 index 0000000000000..c6c648dbb5488 --- /dev/null +++ b/x-pack/plugins/data_enhanced/public/search/utils.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import stringify from 'json-stable-stringify'; + +export async function createRequestHash(keys: Record) { + const msgBuffer = new TextEncoder().encode(stringify(keys)); + const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((b) => ('00' + b.toString(16)).slice(-2)).join(''); +} diff --git a/x-pack/plugins/lens/public/app_plugin/app.tsx b/x-pack/plugins/lens/public/app_plugin/app.tsx index 39163101fc7bd..8caa1737c00ad 100644 --- a/x-pack/plugins/lens/public/app_plugin/app.tsx +++ b/x-pack/plugins/lens/public/app_plugin/app.tsx @@ -82,6 +82,8 @@ export function App({ dashboardFeatureFlag, } = useKibana().services; + const startSession = useCallback(() => data.search.session.start(), [data]); + const [state, setState] = useState(() => { return { query: data.query.queryString.getQuery(), @@ -96,7 +98,7 @@ export function App({ isSaveModalVisible: false, indicateNoData: false, isSaveable: false, - searchSessionId: data.search.session.start(), + searchSessionId: startSession(), }; }); @@ -178,7 +180,7 @@ export function App({ setState((s) => ({ ...s, filters: data.query.filterManager.getFilters(), - searchSessionId: data.search.session.start(), + searchSessionId: startSession(), })); trackUiEvent('app_filters_updated'); }, @@ -188,7 +190,7 @@ export function App({ next: () => { setState((s) => ({ ...s, - searchSessionId: data.search.session.start(), + searchSessionId: startSession(), })); }, }); @@ -199,7 +201,7 @@ export function App({ tap(() => { setState((s) => ({ ...s, - searchSessionId: data.search.session.start(), + searchSessionId: startSession(), })); }), switchMap((done) => @@ -234,6 +236,7 @@ export function App({ data.query, history, initialContext, + startSession, ]); useEffect(() => { @@ -652,7 +655,7 @@ export function App({ // Time change will be picked up by the time subscription setState((s) => ({ ...s, - searchSessionId: data.search.session.start(), + searchSessionId: startSession(), })); trackUiEvent('app_query_change'); } diff --git a/x-pack/test/examples/search_examples/index.ts b/x-pack/test/examples/search_examples/index.ts index 2cac0d1b60de7..13eac7566525e 100644 --- a/x-pack/test/examples/search_examples/index.ts +++ b/x-pack/test/examples/search_examples/index.ts @@ -23,6 +23,7 @@ export default function ({ getService, loadTestFile }: PluginFunctionalProviderC await esArchiver.unload('lens/basic'); }); + loadTestFile(require.resolve('./search_sessions_cache')); loadTestFile(require.resolve('./search_session_example')); }); } diff --git a/x-pack/test/examples/search_examples/search_sessions_cache.ts b/x-pack/test/examples/search_examples/search_sessions_cache.ts new file mode 100644 index 0000000000000..57b2d1665d901 --- /dev/null +++ b/x-pack/test/examples/search_examples/search_sessions_cache.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../functional/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + const PageObjects = getPageObjects(['common']); + const toasts = getService('toasts'); + const retry = getService('retry'); + + async function getExecutedAt() { + const toast = await toasts.getToastElement(1); + const timeElem = await testSubjects.findDescendant('requestExecutedAt', toast); + const text = await timeElem.getVisibleText(); + await toasts.dismissAllToasts(); + await retry.waitFor('toasts gone', async () => { + return (await toasts.getToastCount()) === 0; + }); + return text; + } + + describe.skip('Search session client side cache', () => { + const appId = 'searchExamples'; + + before(async function () { + await PageObjects.common.navigateToApp(appId, { insertTimestamp: false }); + }); + + it('should cache responses by search session id', async () => { + await testSubjects.click('searchExamplesCacheSearch'); + const noSessionExecutedAt = await getExecutedAt(); + + // Expect searches executed in a session to share a response + await testSubjects.click('searchExamplesStartSession'); + await testSubjects.click('searchExamplesCacheSearch'); + const withSessionExecutedAt = await getExecutedAt(); + await testSubjects.click('searchExamplesCacheSearch'); + const withSessionExecutedAt2 = await getExecutedAt(); + expect(withSessionExecutedAt2).to.equal(withSessionExecutedAt); + expect(withSessionExecutedAt).not.to.equal(noSessionExecutedAt); + + // Expect new session to run search again + await testSubjects.click('searchExamplesStartSession'); + await testSubjects.click('searchExamplesCacheSearch'); + const secondSessionExecutedAt = await getExecutedAt(); + expect(secondSessionExecutedAt).not.to.equal(withSessionExecutedAt); + + // Clear session + await testSubjects.click('searchExamplesClearSession'); + await testSubjects.click('searchExamplesCacheSearch'); + const afterClearSession1 = await getExecutedAt(); + await testSubjects.click('searchExamplesCacheSearch'); + const afterClearSession2 = await getExecutedAt(); + expect(secondSessionExecutedAt).not.to.equal(afterClearSession1); + expect(afterClearSession2).not.to.equal(afterClearSession1); + }); + }); +} From 6c7d776fc103a305433ef0bca019aaf7882fb4c6 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Fri, 16 Apr 2021 19:55:16 +0200 Subject: [PATCH 12/25] minimize number of so fild asserted in tests. it creates flakines when implementation details change (#97374) --- .../apis/saved_objects/find.ts | 165 +++--------------- .../apis/saved_objects_management/find.ts | 41 +---- .../saved_objects_management/find.ts | 40 ++--- 3 files changed, 36 insertions(+), 210 deletions(-) diff --git a/test/api_integration/apis/saved_objects/find.ts b/test/api_integration/apis/saved_objects/find.ts index a01562861e606..a4862707e2d0e 100644 --- a/test/api_integration/apis/saved_objects/find.ts +++ b/test/api_integration/apis/saved_objects/find.ts @@ -9,7 +9,6 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { SavedObject } from '../../../../src/core/server'; -import { getKibanaVersion } from './lib/saved_objects_test_utils'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -17,12 +16,6 @@ export default function ({ getService }: FtrProviderContext) { const esDeleteAllIndices = getService('esDeleteAllIndices'); describe('find', () => { - let KIBANA_VERSION: string; - - before(async () => { - KIBANA_VERSION = await getKibanaVersion(getService); - }); - describe('with kibana index', () => { before(() => esArchiver.load('saved_objects/basic')); after(() => esArchiver.unload('saved_objects/basic')); @@ -32,33 +25,9 @@ export default function ({ getService }: FtrProviderContext) { .get('/api/saved_objects/_find?type=visualization&fields=title') .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'visualization', - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - version: 'WzE4LDJd', - attributes: { - title: 'Count of requests', - }, - score: 0, - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - namespaces: ['default'], - references: [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - }, - ], - updated_at: '2017-09-21T18:51:23.794Z', - }, - ], - }); + expect(resp.body.saved_objects.map((so: { id: string }) => so.id)).to.eql([ + 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + ]); expect(resp.body.saved_objects[0].migrationVersion).to.be.ok(); })); @@ -129,33 +98,12 @@ export default function ({ getService }: FtrProviderContext) { .get('/api/saved_objects/_find?type=visualization&fields=title&namespaces=default') .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'visualization', - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - version: 'WzE4LDJd', - attributes: { - title: 'Count of requests', - }, - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - namespaces: ['default'], - score: 0, - references: [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - }, - ], - updated_at: '2017-09-21T18:51:23.794Z', - }, - ], - }); + expect( + resp.body.saved_objects.map((so: { id: string; namespaces: string[] }) => ({ + id: so.id, + namespaces: so.namespaces, + })) + ).to.eql([{ id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', namespaces: ['default'] }]); expect(resp.body.saved_objects[0].migrationVersion).to.be.ok(); })); }); @@ -166,53 +114,15 @@ export default function ({ getService }: FtrProviderContext) { .get('/api/saved_objects/_find?type=visualization&fields=title&namespaces=*') .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 2, - saved_objects: [ - { - type: 'visualization', - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - version: 'WzE4LDJd', - attributes: { - title: 'Count of requests', - }, - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - namespaces: ['default'], - score: 0, - references: [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - }, - ], - updated_at: '2017-09-21T18:51:23.794Z', - }, - { - attributes: { - title: 'Count of requests', - }, - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - namespaces: ['foo-ns'], - references: [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - }, - ], - score: 0, - type: 'visualization', - updated_at: '2017-09-21T18:51:23.794Z', - version: 'WzIyLDJd', - }, - ], - }); + expect( + resp.body.saved_objects.map((so: { id: string; namespaces: string[] }) => ({ + id: so.id, + namespaces: so.namespaces, + })) + ).to.eql([ + { id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', namespaces: ['default'] }, + { id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', namespaces: ['foo-ns'] }, + ]); })); }); @@ -224,42 +134,9 @@ export default function ({ getService }: FtrProviderContext) { ) .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'visualization', - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - attributes: { - title: 'Count of requests', - visState: resp.body.saved_objects[0].attributes.visState, - uiStateJSON: '{"spy":{"mode":{"name":null,"fill":false}}}', - description: '', - version: 1, - kibanaSavedObjectMeta: { - searchSourceJSON: - resp.body.saved_objects[0].attributes.kibanaSavedObjectMeta - .searchSourceJSON, - }, - }, - namespaces: ['default'], - score: 0, - references: [ - { - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - }, - ], - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - updated_at: '2017-09-21T18:51:23.794Z', - version: 'WzE4LDJd', - }, - ], - }); + expect(resp.body.saved_objects.map((so: { id: string }) => so.id)).to.eql([ + 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + ]); })); it('wrong type should return 400 with Bad Request', async () => diff --git a/test/api_integration/apis/saved_objects_management/find.ts b/test/api_integration/apis/saved_objects_management/find.ts index 6ab2352ebb05f..8fb3884a5b37b 100644 --- a/test/api_integration/apis/saved_objects_management/find.ts +++ b/test/api_integration/apis/saved_objects_management/find.ts @@ -34,44 +34,9 @@ export default function ({ getService }: FtrProviderContext) { .get('/api/kibana/management/saved_objects/_find?type=visualization&fields=title') .expect(200) .then((resp: Response) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'visualization', - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - version: 'WzE4LDJd', - attributes: { - title: 'Count of requests', - }, - migrationVersion: resp.body.saved_objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - namespaces: ['default'], - references: [ - { - id: '91200a00-9efd-11e7-acb3-3dab96693fab', - name: 'kibanaSavedObjectMeta.searchSourceJSON.index', - type: 'index-pattern', - }, - ], - score: 0, - updated_at: '2017-09-21T18:51:23.794Z', - meta: { - editUrl: - '/management/kibana/objects/savedVisualizations/dd7caf20-9efd-11e7-acb3-3dab96693fab', - icon: 'visualizeApp', - inAppUrl: { - path: '/app/visualize#/edit/dd7caf20-9efd-11e7-acb3-3dab96693fab', - uiCapabilitiesPath: 'visualize.show', - }, - title: 'Count of requests', - namespaceType: 'single', - }, - }, - ], - }); + expect(resp.body.saved_objects.map((so: { id: string }) => so.id)).to.eql([ + 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + ]); })); describe('unknown type', () => { diff --git a/test/plugin_functional/test_suites/saved_objects_management/find.ts b/test/plugin_functional/test_suites/saved_objects_management/find.ts index 5dce8f43339a1..e5a5d69c7e4d4 100644 --- a/test/plugin_functional/test_suites/saved_objects_management/find.ts +++ b/test/plugin_functional/test_suites/saved_objects_management/find.ts @@ -33,28 +33,17 @@ export default function ({ getService }: PluginFunctionalProviderContext) { .set('kbn-xsrf', 'true') .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 1, - saved_objects: [ - { - type: 'test-hidden-importable-exportable', - id: 'ff3733a0-9fty-11e7-ahb3-3dcb94193fab', - attributes: { - title: 'Hidden Saved object type that is importable/exportable.', - }, - references: [], - updated_at: '2021-02-11T18:51:23.794Z', - version: 'WzIsMl0=', - namespaces: ['default'], - score: 0, - meta: { - namespaceType: 'single', - }, - }, - ], - }); + expect( + resp.body.saved_objects.map((so: { id: string; type: string }) => ({ + id: so.id, + type: so.type, + })) + ).to.eql([ + { + type: 'test-hidden-importable-exportable', + id: 'ff3733a0-9fty-11e7-ahb3-3dcb94193fab', + }, + ]); })); it('returns empty response for non importableAndExportable types', async () => @@ -65,12 +54,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { .set('kbn-xsrf', 'true') .expect(200) .then((resp) => { - expect(resp.body).to.eql({ - page: 1, - per_page: 20, - total: 0, - saved_objects: [], - }); + expect(resp.body.saved_objects).to.eql([]); })); }); }); From 194355fdd3969f567f43ad4b7f63d72dcf7974a9 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Fri, 16 Apr 2021 10:57:29 -0700 Subject: [PATCH 13/25] Skip test to try and stabilize master https://github.com/elastic/kibana/issues/97378 Signed-off-by: Tyler Smalley --- .../apis/security_solution/matrix_dns_histogram.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security_solution/matrix_dns_histogram.ts b/x-pack/test/api_integration/apis/security_solution/matrix_dns_histogram.ts index 69beb65dec670..27a7a5a539607 100644 --- a/x-pack/test/api_integration/apis/security_solution/matrix_dns_histogram.ts +++ b/x-pack/test/api_integration/apis/security_solution/matrix_dns_histogram.ts @@ -33,7 +33,8 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const retry = getService('retry'); - describe('Matrix DNS Histogram', () => { + // FIX: https://github.com/elastic/kibana/issues/97378 + describe.skip('Matrix DNS Histogram', () => { describe('Large data set', () => { before(() => esArchiver.load('security_solution/matrix_dns_histogram/large_dns_query')); after(() => esArchiver.unload('security_solution/matrix_dns_histogram/large_dns_query')); From 1cbdb26ceacb47f87adc3a83ae516c08f6fa1d0e Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 16 Apr 2021 12:11:12 -0700 Subject: [PATCH 14/25] skip flaky suite (#97355) --- .../api_integration/apis/security_solution/feature_controls.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/security_solution/feature_controls.ts b/x-pack/test/api_integration/apis/security_solution/feature_controls.ts index 1e43fd473a38d..da28e28dae769 100644 --- a/x-pack/test/api_integration/apis/security_solution/feature_controls.ts +++ b/x-pack/test/api_integration/apis/security_solution/feature_controls.ts @@ -58,7 +58,8 @@ export default function ({ getService }: FtrProviderContext) { }; }; - describe('feature controls', () => { + // FLAKY: https://github.com/elastic/kibana/issues/97355 + describe.skip('feature controls', () => { it(`APIs can't be accessed by user with no privileges`, async () => { const username = 'logstash_read'; const roleName = 'logstash_read'; From 721f4b55f506d48cad34fe38afb351dda3367a25 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 16 Apr 2021 13:52:35 -0600 Subject: [PATCH 15/25] [Security Solutions] Fixes flake with cypress tests (#97329) ## Summary Fixes some recent flakeyness with Cypress tests * Adds cypress.pipe() on button clicks around the area of flakes * Adds an alerting threshold to the utilities so we can wait for when an exact number of alerts are available on a page * Changes the alerts to not run again with 10 seconds, because if a test takes longer than 10 seconds, the rule can run a second time which can invalidate some of the text when running checks when timeline or other components update on their button clicks. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../detection_alerts/attach_to_case.spec.ts | 2 +- .../detection_alerts/closing.spec.ts | 4 ++-- .../detection_alerts/in_progress.spec.ts | 2 +- .../investigate_in_timeline.spec.ts | 2 +- .../detection_alerts/opening.spec.ts | 2 +- .../integration/exceptions/from_alert.spec.ts | 2 +- .../integration/exceptions/from_rule.spec.ts | 2 +- .../security_solution/cypress/objects/rule.ts | 4 ++-- .../security_solution/cypress/tasks/alerts.ts | 20 +++++++++++++++---- .../cypress/tasks/api_calls/rules.ts | 15 +++++++++----- .../cypress/tasks/create_new_rule.ts | 4 ++-- 11 files changed, 38 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts index e63ef513cc638..bdf2ab96600ea 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts @@ -32,7 +32,7 @@ describe('Alerts timeline', () => { waitForAlertsIndexToBeCreated(); createCustomRuleActivated(newRule); refreshPage(); - waitForAlertsToPopulate(); + waitForAlertsToPopulate(500); // Then we login as read-only user to test. login(ROLES.reader); diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts index b7c0e1c6fcd6e..741f05129f9c4 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/closing.spec.ts @@ -39,9 +39,9 @@ describe('Closing alerts', () => { loginAndWaitForPage(DETECTIONS_URL); waitForAlertsPanelToBeLoaded(); waitForAlertsIndexToBeCreated(); - createCustomRuleActivated(newRule); + createCustomRuleActivated(newRule, '1', '100m', 100); refreshPage(); - waitForAlertsToPopulate(); + waitForAlertsToPopulate(100); deleteCustomRule(); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts index 8efdbe82c3492..b4f890e4d8dbf 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/in_progress.spec.ts @@ -38,7 +38,7 @@ describe('Marking alerts as in-progress', () => { waitForAlertsIndexToBeCreated(); createCustomRuleActivated(newRule); refreshPage(); - waitForAlertsToPopulate(); + waitForAlertsToPopulate(500); }); it('Mark one alert in progress when more than one open alerts are selected', () => { diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts index bc4929cd1341d..d705cb652d2ea 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/investigate_in_timeline.spec.ts @@ -29,7 +29,7 @@ describe('Alerts timeline', () => { waitForAlertsIndexToBeCreated(); createCustomRuleActivated(newRule); refreshPage(); - waitForAlertsToPopulate(); + waitForAlertsToPopulate(500); }); it('Investigate alert in default timeline', () => { diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts index ec0923beb4c40..bc907dccd0a04 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/opening.spec.ts @@ -39,7 +39,7 @@ describe('Opening alerts', () => { waitForAlertsIndexToBeCreated(); createCustomRuleActivated(newRule); refreshPage(); - waitForAlertsToPopulate(); + waitForAlertsToPopulate(500); selectNumberOfAlerts(5); cy.get(SELECTED_ALERTS).should('have.text', `Selected 5 alerts`); diff --git a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts index d5e0b56b8e267..e36809380df86 100644 --- a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_alert.spec.ts @@ -43,7 +43,7 @@ describe('From alert', () => { cleanKibana(); loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); waitForAlertsIndexToBeCreated(); - createCustomRule(newRule); + createCustomRule(newRule, 'rule_testing', '10s'); goToManageAlertsDetectionRules(); goToRuleDetails(); diff --git a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts index 148254a813b56..e0d7e5a32edfd 100644 --- a/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/exceptions/from_rule.spec.ts @@ -41,7 +41,7 @@ describe('From rule', () => { cleanKibana(); loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); waitForAlertsIndexToBeCreated(); - createCustomRule(newRule); + createCustomRule(newRule, 'rule_testing', '10s'); goToManageAlertsDetectionRules(); goToRuleDetails(); diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index f083cc5da6f53..099cd39ba2d7b 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -185,7 +185,7 @@ export const existingRule: CustomRule = { name: 'Rule 1', description: 'Description for Rule 1', index: ['auditbeat-*'], - interval: '10s', + interval: '100m', severity: 'High', riskScore: '19', tags: ['rule1'], @@ -332,5 +332,5 @@ export const editedRule = { export const expectedExportedRule = (ruleResponse: Cypress.Response) => { const jsonrule = ruleResponse.body; - return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"10s","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-17520h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n`; + return `{"id":"${jsonrule.id}","updated_at":"${jsonrule.updated_at}","updated_by":"elastic","created_at":"${jsonrule.created_at}","created_by":"elastic","name":"${jsonrule.name}","tags":[],"interval":"100m","enabled":false,"description":"${jsonrule.description}","risk_score":${jsonrule.risk_score},"severity":"${jsonrule.severity}","output_index":".siem-signals-default","author":[],"false_positives":[],"from":"now-17520h","rule_id":"rule_testing","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"type":"query","language":"kuery","index":["exceptions-*"],"query":"${jsonrule.query}","throttle":"no_actions","actions":[]}\n{"exported_count":1,"missing_rules":[],"missing_rules_count":0}\n`; }; diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts index dd7a163d00753..b677e36ab3918 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts @@ -35,13 +35,25 @@ export const addExceptionFromFirstAlert = () => { }; export const closeFirstAlert = () => { - cy.get(TIMELINE_CONTEXT_MENU_BTN).first().click({ force: true }); - cy.get(CLOSE_ALERT_BTN).click(); + cy.get(TIMELINE_CONTEXT_MENU_BTN) + .first() + .pipe(($el) => $el.trigger('click')) + .should('be.visible'); + + cy.get(CLOSE_ALERT_BTN) + .pipe(($el) => $el.trigger('click')) + .should('not.be.visible'); }; export const closeAlerts = () => { - cy.get(TAKE_ACTION_POPOVER_BTN).click({ force: true }); - cy.get(CLOSE_SELECTED_ALERTS_BTN).click(); + cy.get(TAKE_ACTION_POPOVER_BTN) + .first() + .pipe(($el) => $el.trigger('click')) + .should('be.visible'); + + cy.get(CLOSE_SELECTED_ALERTS_BTN) + .pipe(($el) => $el.trigger('click')) + .should('not.be.visible'); }; export const expandFirstAlert = () => { diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts index 0b051f3a26581..5a816a71744cb 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts @@ -7,7 +7,7 @@ import { CustomRule, ThreatIndicatorRule } from '../../objects/rule'; -export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing') => +export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing', interval = '100m') => cy.request({ method: 'POST', url: 'api/detection_engine/rules', @@ -15,7 +15,7 @@ export const createCustomRule = (rule: CustomRule, ruleId = 'rule_testing') => rule_id: ruleId, risk_score: parseInt(rule.riskScore, 10), description: rule.description, - interval: '10s', + interval, name: rule.name, severity: rule.severity.toLocaleLowerCase(), type: 'query', @@ -67,7 +67,12 @@ export const createCustomIndicatorRule = (rule: ThreatIndicatorRule, ruleId = 'r failOnStatusCode: false, }); -export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') => +export const createCustomRuleActivated = ( + rule: CustomRule, + ruleId = '1', + interval = '100m', + maxSignals = 500 +) => cy.request({ method: 'POST', url: 'api/detection_engine/rules', @@ -75,7 +80,7 @@ export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') => rule_id: ruleId, risk_score: parseInt(rule.riskScore, 10), description: rule.description, - interval: '10s', + interval, name: rule.name, severity: rule.severity.toLocaleLowerCase(), type: 'query', @@ -85,7 +90,7 @@ export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') => language: 'kuery', enabled: true, tags: ['rule1'], - max_signals: 500, + max_signals: maxSignals, }, headers: { 'kbn-xsrf': 'cypress-creds' }, failOnStatusCode: false, diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts index 2b7308757f9f4..9f957a0cb9a95 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts @@ -479,7 +479,7 @@ export const selectThresholdRuleType = () => { cy.get(THRESHOLD_TYPE).click({ force: true }); }; -export const waitForAlertsToPopulate = async () => { +export const waitForAlertsToPopulate = async (alertCountThreshold = 1) => { cy.waitUntil( () => { refreshPage(); @@ -488,7 +488,7 @@ export const waitForAlertsToPopulate = async () => { .invoke('text') .then((countText) => { const alertCount = parseInt(countText, 10) || 0; - return alertCount > 0; + return alertCount >= alertCountThreshold; }); }, { interval: 500, timeout: 12000 } From e321f57f64657ffff91df8ed96f4e9fdbe5dcde7 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 16 Apr 2021 22:28:19 -0700 Subject: [PATCH 16/25] skip flaky suite (#97382) --- .../test/api_integration/apis/short_urls/feature_controls.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/short_urls/feature_controls.ts b/x-pack/test/api_integration/apis/short_urls/feature_controls.ts index a2596e9eaedaf..e55fcf10b7fac 100644 --- a/x-pack/test/api_integration/apis/short_urls/feature_controls.ts +++ b/x-pack/test/api_integration/apis/short_urls/feature_controls.ts @@ -12,7 +12,8 @@ export default function featureControlsTests({ getService }: FtrProviderContext) const supertest = getService('supertestWithoutAuth'); const security = getService('security'); - describe('feature controls', () => { + // FLAKY: https://github.com/elastic/kibana/issues/97382 + describe.skip('feature controls', () => { const kibanaUsername = 'kibana_admin'; const kibanaUserRoleName = 'kibana_admin'; From e0da8b2e961793b6df66086769a0b2f830186d2d Mon Sep 17 00:00:00 2001 From: Bryan Clement Date: Sat, 17 Apr 2021 03:42:49 -0700 Subject: [PATCH 17/25] [Asset Management] Agent picker follow up (#97357) --- .../osquery/public/agents/agent_grouper.ts | 118 ++++++++++ .../osquery/public/agents/agents_table.tsx | 217 ++++++------------ .../osquery/public/agents/helpers.test.ts | 6 + .../plugins/osquery/public/agents/helpers.ts | 65 +++++- .../osquery/public/agents/translations.ts | 2 +- x-pack/plugins/osquery/public/agents/types.ts | 13 +- .../osquery/public/agents/use_agent_groups.ts | 14 +- .../public/agents/use_agent_policies.ts | 38 +++ .../osquery/public/agents/use_all_agents.ts | 24 +- .../live_query/form/agents_table_field.tsx | 5 +- 10 files changed, 344 insertions(+), 158 deletions(-) create mode 100644 x-pack/plugins/osquery/public/agents/agent_grouper.ts create mode 100644 x-pack/plugins/osquery/public/agents/use_agent_policies.ts diff --git a/x-pack/plugins/osquery/public/agents/agent_grouper.ts b/x-pack/plugins/osquery/public/agents/agent_grouper.ts new file mode 100644 index 0000000000000..419a3b9e733a4 --- /dev/null +++ b/x-pack/plugins/osquery/public/agents/agent_grouper.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Agent } from '../../common/shared_imports'; +import { generateColorPicker } from './helpers'; +import { + ALL_AGENTS_LABEL, + AGENT_PLATFORMS_LABEL, + AGENT_POLICY_LABEL, + AGENT_SELECTION_LABEL, +} from './translations'; +import { AGENT_GROUP_KEY, Group, GroupOption } from './types'; + +const getColor = generateColorPicker(); + +const generateGroup = (label: string, groupType: AGENT_GROUP_KEY) => { + return { + label, + groupType, + color: getColor(groupType), + size: 0, + data: [] as T[], + }; +}; + +export class AgentGrouper { + groupOrder = [ + AGENT_GROUP_KEY.All, + AGENT_GROUP_KEY.Platform, + AGENT_GROUP_KEY.Policy, + AGENT_GROUP_KEY.Agent, + ]; + groups = { + [AGENT_GROUP_KEY.All]: generateGroup(ALL_AGENTS_LABEL, AGENT_GROUP_KEY.All), + [AGENT_GROUP_KEY.Platform]: generateGroup(AGENT_PLATFORMS_LABEL, AGENT_GROUP_KEY.Platform), + [AGENT_GROUP_KEY.Policy]: generateGroup(AGENT_POLICY_LABEL, AGENT_GROUP_KEY.Policy), + [AGENT_GROUP_KEY.Agent]: generateGroup(AGENT_SELECTION_LABEL, AGENT_GROUP_KEY.Agent), + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + updateGroup(key: AGENT_GROUP_KEY, data: any[], append = false) { + if (!data?.length) { + return; + } + const group = this.groups[key]; + if (append) { + group.data.push(...data); + } else { + group.data = data; + } + group.size = data.length; + } + + setTotalAgents(total: number): void { + this.groups[AGENT_GROUP_KEY.All].size = total; + } + + generateOptions(): GroupOption[] { + const opts: GroupOption[] = []; + for (const key of this.groupOrder) { + const { label, size, groupType, data, color } = this.groups[key]; + if (size === 0) { + continue; + } + + switch (key) { + case AGENT_GROUP_KEY.All: + opts.push({ + label, + options: [ + { + label, + value: { groupType, size }, + color, + }, + ], + }); + break; + case AGENT_GROUP_KEY.Platform: + case AGENT_GROUP_KEY.Policy: + opts.push({ + label, + options: (data as Group[]).map(({ name, id, size: groupSize }) => ({ + label: name !== id ? `${name} (${id})` : name, + key: id, + color: getColor(groupType), + value: { groupType, id, size: groupSize }, + })), + }); + break; + case AGENT_GROUP_KEY.Agent: + opts.push({ + label, + options: (data as Agent[]).map((agent: Agent) => ({ + label: `${agent.local_metadata.host.hostname} (${agent.local_metadata.elastic.agent.id})`, + key: agent.local_metadata.elastic.agent.id, + color, + value: { + groupType, + groups: { + policy: agent.policy_id ?? '', + platform: agent.local_metadata.os.platform, + }, + id: agent.local_metadata.elastic.agent.id, + online: agent.active, + }, + })), + }); + break; + } + } + return opts; + } +} diff --git a/x-pack/plugins/osquery/public/agents/agents_table.tsx b/x-pack/plugins/osquery/public/agents/agents_table.tsx index 5f1b6a0d2f0b1..38132957c341f 100644 --- a/x-pack/plugins/osquery/public/agents/agents_table.tsx +++ b/x-pack/plugins/osquery/public/agents/agents_table.tsx @@ -5,179 +5,98 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; -import { EuiComboBox, EuiComboBoxOptionOption, EuiHealth, EuiHighlight } from '@elastic/eui'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { EuiComboBox, EuiHealth, EuiHighlight } from '@elastic/eui'; +import { useDebounce } from 'react-use'; import { useAllAgents } from './use_all_agents'; import { useAgentGroups } from './use_agent_groups'; import { useOsqueryPolicies } from './use_osquery_policies'; -import { Agent } from '../../common/shared_imports'; +import { AgentGrouper } from './agent_grouper'; import { getNumAgentsInGrouping, generateAgentCheck, getNumOverlapped, - generateColorPicker, + generateAgentSelection, } from './helpers'; -import { - ALL_AGENTS_LABEL, - AGENT_PLATFORMS_LABEL, - AGENT_POLICY_LABEL, - SELECT_AGENT_LABEL, - AGENT_SELECTION_LABEL, - generateSelectedAgentsMessage, -} from './translations'; - -import { AGENT_GROUP_KEY, SelectedGroups, AgentOptionValue, GroupOptionValue } from './types'; +import { SELECT_AGENT_LABEL, generateSelectedAgentsMessage } from './translations'; -export interface AgentsSelection { - agents: string[]; - allAgentsSelected: boolean; - platformsSelected: string[]; - policiesSelected: string[]; -} +import { + AGENT_GROUP_KEY, + SelectedGroups, + AgentOptionValue, + GroupOption, + AgentSelection, +} from './types'; interface AgentsTableProps { - agentSelection: AgentsSelection; - onChange: (payload: AgentsSelection) => void; + agentSelection: AgentSelection; + onChange: (payload: AgentSelection) => void; } -type GroupOption = EuiComboBoxOptionOption; - -const getColor = generateColorPicker(); +const perPage = 10; +const DEBOUNCE_DELAY = 100; // ms const AgentsTableComponent: React.FC = ({ onChange }) => { + // search related + const [searchValue, setSearchValue] = useState(''); + const [modifyingSearch, setModifyingSearch] = useState(false); + const [debouncedSearchValue, setDebouncedSearchValue] = useState(''); + useDebounce( + () => { + // update the real search value, set the typing flag + setDebouncedSearchValue(searchValue); + setModifyingSearch(false); + }, + DEBOUNCE_DELAY, + [searchValue] + ); + + // grouping related const osqueryPolicyData = useOsqueryPolicies(); const { loading: groupsLoading, totalCount: totalNumAgents, groups } = useAgentGroups( osqueryPolicyData ); - const { agents } = useAllAgents(osqueryPolicyData); - const [loading, setLoading] = useState(true); + const grouper = useMemo(() => new AgentGrouper(), []); + const { agentsLoading, agents } = useAllAgents(osqueryPolicyData, debouncedSearchValue, { + perPage, + }); + + // option related const [options, setOptions] = useState([]); const [selectedOptions, setSelectedOptions] = useState([]); const [numAgentsSelected, setNumAgentsSelected] = useState(0); useEffect(() => { - const allAgentsLabel = ALL_AGENTS_LABEL; - const opts: GroupOption[] = [ - { - label: allAgentsLabel, - options: [ - { - label: allAgentsLabel, - value: { groupType: AGENT_GROUP_KEY.All, size: totalNumAgents }, - color: getColor(AGENT_GROUP_KEY.All), - }, - ], - }, - ]; - - if (groups.platforms.length > 0) { - const groupType = AGENT_GROUP_KEY.Platform; - opts.push({ - label: AGENT_PLATFORMS_LABEL, - options: groups.platforms.map(({ name, size }) => ({ - label: name, - color: getColor(groupType), - value: { groupType, size }, - })), - }); - } - - if (groups.policies.length > 0) { - const groupType = AGENT_GROUP_KEY.Policy; - opts.push({ - label: AGENT_POLICY_LABEL, - options: groups.policies.map(({ name, size }) => ({ - label: name, - color: getColor(groupType), - value: { groupType, size }, - })), - }); - } - - if (agents && agents.length > 0) { - const groupType = AGENT_GROUP_KEY.Agent; - opts.push({ - label: AGENT_SELECTION_LABEL, - options: (agents as Agent[]).map((agent: Agent) => ({ - label: agent.local_metadata.host.hostname, - color: getColor(groupType), - value: { - groupType, - groups: { policy: agent.policy_id ?? '', platform: agent.local_metadata.os.platform }, - id: agent.local_metadata.elastic.agent.id, - online: agent.active, - }, - })), - }); - } - setLoading(false); - setOptions(opts); - }, [groups.platforms, groups.policies, totalNumAgents, groupsLoading, agents]); + // update the groups when groups or agents have changed + grouper.setTotalAgents(totalNumAgents); + grouper.updateGroup(AGENT_GROUP_KEY.Platform, groups.platforms); + grouper.updateGroup(AGENT_GROUP_KEY.Policy, groups.policies); + grouper.updateGroup(AGENT_GROUP_KEY.Agent, agents); + const newOptions = grouper.generateOptions(); + setOptions(newOptions); + }, [groups.platforms, groups.policies, totalNumAgents, groupsLoading, agents, grouper]); const onSelection = useCallback( (selection: GroupOption[]) => { - // TODO?: optimize this by making it incremental - const newAgentSelection: AgentsSelection = { - agents: [], - allAgentsSelected: false, - platformsSelected: [], - policiesSelected: [], - }; - // parse through the selections to be able to determine how many are actually selected - const selectedAgents = []; - const selectedGroups: SelectedGroups = { - policy: {}, - platform: {}, - }; - - // TODO: clean this up, make it less awkward - for (const opt of selection) { - const groupType = opt.value?.groupType; - let value; - switch (groupType) { - case AGENT_GROUP_KEY.All: - newAgentSelection.allAgentsSelected = true; - break; - case AGENT_GROUP_KEY.Platform: - value = opt.value as GroupOptionValue; - if (!newAgentSelection.allAgentsSelected) { - // we don't need to calculate diffs when all agents are selected - selectedGroups.platform[opt.label] = value.size; - } - newAgentSelection.platformsSelected.push(opt.label); - break; - case AGENT_GROUP_KEY.Policy: - value = opt.value as GroupOptionValue; - if (!newAgentSelection.allAgentsSelected) { - // we don't need to calculate diffs when all agents are selected - selectedGroups.policy[opt.label] = value.size ?? 0; - } - newAgentSelection.policiesSelected.push(opt.label); - break; - case AGENT_GROUP_KEY.Agent: - value = opt.value as AgentOptionValue; - if (!newAgentSelection.allAgentsSelected) { - // we don't need to count how many agents are selected if they are all selected - selectedAgents.push(opt.value); - } - // TODO: fix this casting by updating the opt type to be a union - newAgentSelection.agents.push(value.id as string); - break; - default: - // this should never happen! - // eslint-disable-next-line no-console - console.error(`unknown group type ${groupType}`); - } - } + // TODO?: optimize this by making the selection computation incremental + const { + newAgentSelection, + selectedAgents, + selectedGroups, + }: { + newAgentSelection: AgentSelection; + selectedAgents: AgentOptionValue[]; + selectedGroups: SelectedGroups; + } = generateAgentSelection(selection); if (newAgentSelection.allAgentsSelected) { setNumAgentsSelected(totalNumAgents); } else { const checkAgent = generateAgentCheck(selectedGroups); setNumAgentsSelected( // filter out all the agents counted by selected policies and platforms - selectedAgents.filter((a) => checkAgent(a as AgentOptionValue)).length + + selectedAgents.filter(checkAgent).length + // add the number of agents added via policy and platform groups getNumAgentsInGrouping(selectedGroups) - // subtract the number of agents double counted by policy/platform selections @@ -190,32 +109,40 @@ const AgentsTableComponent: React.FC = ({ onChange }) => { [groups, onChange, totalNumAgents] ); - const renderOption = useCallback((option, searchValue, contentClassName) => { + const renderOption = useCallback((option, searchVal, contentClassName) => { const { label, value } = option; return value?.groupType === AGENT_GROUP_KEY.Agent ? ( - {label} + {label} ) : ( - {label} + [{value?.size ?? 0}]   - ({value?.size}) + {label} ); }, []); + + const onSearchChange = useCallback((v: string) => { + // set the typing flag and update the search value + setModifyingSearch(v !== ''); + setSearchValue(v); + }, []); + return (
-

{SELECT_AGENT_LABEL}

{numAgentsSelected > 0 ? {generateSelectedAgentsMessage(numAgentsSelected)} : ''}   { const { platforms, policies, overlap } = processAggregations(input); expect(platforms).toEqual([ { + id: 'darwin', name: 'darwin', size: 200, }, @@ -59,10 +60,12 @@ describe('processAggregations', () => { expect(platforms).toEqual([]); expect(policies).toEqual([ { + id: '8cd01a60-8a74-11eb-86cb-c58693443a4f', name: '8cd01a60-8a74-11eb-86cb-c58693443a4f', size: 100, }, { + id: '8cd06880-8a74-11eb-86cb-c58693443a4f', name: '8cd06880-8a74-11eb-86cb-c58693443a4f', size: 100, }, @@ -107,16 +110,19 @@ describe('processAggregations', () => { const { platforms, policies, overlap } = processAggregations(input); expect(platforms).toEqual([ { + id: 'darwin', name: 'darwin', size: 200, }, ]); expect(policies).toEqual([ { + id: '8cd01a60-8a74-11eb-86cb-c58693443a4f', name: '8cd01a60-8a74-11eb-86cb-c58693443a4f', size: 100, }, { + id: '8cd06880-8a74-11eb-86cb-c58693443a4f', name: '8cd06880-8a74-11eb-86cb-c58693443a4f', size: 100, }, diff --git a/x-pack/plugins/osquery/public/agents/helpers.ts b/x-pack/plugins/osquery/public/agents/helpers.ts index 830fca5f57caa..14a8dd64fb4da 100644 --- a/x-pack/plugins/osquery/public/agents/helpers.ts +++ b/x-pack/plugins/osquery/public/agents/helpers.ts @@ -20,6 +20,9 @@ import { Group, AgentOptionValue, AggregationDataPoint, + AgentSelection, + GroupOptionValue, + GroupOption, } from './types'; export type InspectResponse = Inspect & { response: string[] }; @@ -43,11 +46,12 @@ export const processAggregations = (aggs: Record) => { const platformTerms = aggs.platforms as TermsAggregate; const policyTerms = aggs.policies as TermsAggregate; - const policies = policyTerms?.buckets.map((o) => ({ name: o.key, size: o.doc_count })) ?? []; + const policies = + policyTerms?.buckets.map((o) => ({ name: o.key, id: o.key, size: o.doc_count })) ?? []; if (platformTerms?.buckets) { for (const { key, doc_count: size, policies: platformPolicies } of platformTerms.buckets) { - platforms.push({ name: key, size }); + platforms.push({ name: key, id: key, size }); if (platformPolicies?.buckets && policies.length > 0) { overlap[key] = platformPolicies.buckets.reduce((acc: { [key: string]: number }, pol) => { acc[pol.key] = pol.doc_count; @@ -96,6 +100,63 @@ export const generateAgentCheck = (selectedGroups: SelectedGroups) => { }; }; +export const generateAgentSelection = (selection: GroupOption[]) => { + const newAgentSelection: AgentSelection = { + agents: [], + allAgentsSelected: false, + platformsSelected: [], + policiesSelected: [], + }; + // parse through the selections to be able to determine how many are actually selected + const selectedAgents: AgentOptionValue[] = []; + const selectedGroups: SelectedGroups = { + policy: {}, + platform: {}, + }; + + // TODO: clean this up, make it less awkward + for (const opt of selection) { + const groupType = opt.value?.groupType; + let value; + switch (groupType) { + case AGENT_GROUP_KEY.All: + newAgentSelection.allAgentsSelected = true; + break; + case AGENT_GROUP_KEY.Platform: + value = opt.value as GroupOptionValue; + if (!newAgentSelection.allAgentsSelected) { + // we don't need to calculate diffs when all agents are selected + selectedGroups.platform[opt.value?.id ?? opt.label] = value.size; + } + newAgentSelection.platformsSelected.push(opt.label); + break; + case AGENT_GROUP_KEY.Policy: + value = opt.value as GroupOptionValue; + if (!newAgentSelection.allAgentsSelected) { + // we don't need to calculate diffs when all agents are selected + selectedGroups.policy[opt.value?.id ?? opt.label] = value.size; + } + newAgentSelection.policiesSelected.push(opt.label); + break; + case AGENT_GROUP_KEY.Agent: + value = opt.value as AgentOptionValue; + if (!newAgentSelection.allAgentsSelected) { + // we don't need to count how many agents are selected if they are all selected + selectedAgents.push(value); + } + if (value?.id) { + newAgentSelection.agents.push(value.id); + } + break; + default: + // this should never happen! + // eslint-disable-next-line no-console + console.error(`unknown group type ${groupType}`); + } + } + return { newAgentSelection, selectedGroups, selectedAgents }; +}; + export const generateTablePaginationOptions = ( activePage: number, limit: number, diff --git a/x-pack/plugins/osquery/public/agents/translations.ts b/x-pack/plugins/osquery/public/agents/translations.ts index af99a73d63de2..209761b4c8bdf 100644 --- a/x-pack/plugins/osquery/public/agents/translations.ts +++ b/x-pack/plugins/osquery/public/agents/translations.ts @@ -40,7 +40,7 @@ export const AGENT_SELECTION_LABEL = i18n.translate('xpack.osquery.agents.select }); export const SELECT_AGENT_LABEL = i18n.translate('xpack.osquery.agents.selectAgentLabel', { - defaultMessage: `Select Agents`, + defaultMessage: `Select agents or groups`, }); export const ERROR_ALL_AGENTS = i18n.translate('xpack.osquery.agents.errorSearchDescription', { diff --git a/x-pack/plugins/osquery/public/agents/types.ts b/x-pack/plugins/osquery/public/agents/types.ts index 2fa8ddaf345cd..b26404f9c5e70 100644 --- a/x-pack/plugins/osquery/public/agents/types.ts +++ b/x-pack/plugins/osquery/public/agents/types.ts @@ -6,6 +6,7 @@ */ import { TermsAggregate } from '@elastic/elasticsearch/api/types'; +import { EuiComboBoxOptionOption } from '@elastic/eui'; interface BaseDataPoint { key: string; @@ -17,6 +18,7 @@ export type AggregationDataPoint = BaseDataPoint & { }; export interface Group { + id: string; name: string; size: number; } @@ -28,14 +30,23 @@ export interface SelectedGroups { [groupType: string]: { [groupName: string]: number }; } +export type GroupOption = EuiComboBoxOptionOption; + +export interface AgentSelection { + agents: string[]; + allAgentsSelected: boolean; + platformsSelected: string[]; + policiesSelected: string[]; +} + interface BaseGroupOption { + id?: string; groupType: AGENT_GROUP_KEY; } export type AgentOptionValue = BaseGroupOption & { groups: { [groupType: string]: string }; online: boolean; - id: string; }; export type GroupOptionValue = BaseGroupOption & { diff --git a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts index 0eaca65d02d4b..0853891f1919d 100644 --- a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts +++ b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts @@ -7,6 +7,7 @@ import { useState } from 'react'; import { useQuery } from 'react-query'; import { useKibana } from '../common/lib/kibana'; +import { useAgentPolicies } from './use_agent_policies'; import { OsqueryQueries, @@ -25,6 +26,7 @@ interface UseAgentGroups { export const useAgentGroups = ({ osqueryPolicies, osqueryPoliciesLoading }: UseAgentGroups) => { const { data } = useKibana().services; + const { agentPoliciesLoading, agentPolicyById } = useAgentPolicies(osqueryPolicies); const [platforms, setPlatforms] = useState([]); const [policies, setPolicies] = useState([]); const [loading, setLoading] = useState(true); @@ -78,14 +80,22 @@ export const useAgentGroups = ({ osqueryPolicies, osqueryPoliciesLoading }: UseA setPlatforms(newPlatforms); setOverlap(newOverlap); - setPolicies(newPolicies); + setPolicies( + newPolicies.map((p) => { + const name = agentPolicyById[p.id]?.name ?? p.name; + return { + ...p, + name, + }; + }) + ); } setLoading(false); setTotalCount(responseData.totalCount); }, { - enabled: !osqueryPoliciesLoading, + enabled: !osqueryPoliciesLoading && !agentPoliciesLoading, } ); diff --git a/x-pack/plugins/osquery/public/agents/use_agent_policies.ts b/x-pack/plugins/osquery/public/agents/use_agent_policies.ts new file mode 100644 index 0000000000000..3045423ccbe2d --- /dev/null +++ b/x-pack/plugins/osquery/public/agents/use_agent_policies.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQueries, UseQueryResult } from 'react-query'; +import { useKibana } from '../common/lib/kibana'; +import { + AgentPolicy, + agentPolicyRouteService, + GetOneAgentPolicyResponse, +} from '../../../fleet/common'; + +export const useAgentPolicies = (policyIds: string[] = []) => { + const { http } = useKibana().services; + + const agentResponse = useQueries( + policyIds.map((policyId) => ({ + queryKey: ['agentPolicy', policyId], + queryFn: () => http.get(agentPolicyRouteService.getInfoPath(policyId)), + enabled: policyIds.length > 0, + })) + ) as Array>; + + const agentPoliciesLoading = agentResponse.some((p) => p.isLoading); + const agentPolicies = agentResponse.map((p) => p.data?.item); + const agentPolicyById = agentPolicies.reduce((acc, p) => { + if (!p) { + return acc; + } + acc[p.id] = p; + return acc; + }, {} as { [key: string]: AgentPolicy }); + + return { agentPoliciesLoading, agentPolicies, agentPolicyById }; +}; diff --git a/x-pack/plugins/osquery/public/agents/use_all_agents.ts b/x-pack/plugins/osquery/public/agents/use_all_agents.ts index 607f9ae007692..bd9b1c32412e6 100644 --- a/x-pack/plugins/osquery/public/agents/use_all_agents.ts +++ b/x-pack/plugins/osquery/public/agents/use_all_agents.ts @@ -14,16 +14,30 @@ interface UseAllAgents { osqueryPoliciesLoading: boolean; } -export const useAllAgents = ({ osqueryPolicies, osqueryPoliciesLoading }: UseAllAgents) => { - // TODO: properly fetch these in an async manner +interface RequestOptions { + perPage?: number; + page?: number; +} + +// TODO: break out the paginated vs all cases into separate hooks +export const useAllAgents = ( + { osqueryPolicies, osqueryPoliciesLoading }: UseAllAgents, + searchValue = '', + opts: RequestOptions = { perPage: 9000 } +) => { + const { perPage } = opts; const { http } = useKibana().services; const { isLoading: agentsLoading, data: agentData } = useQuery( - ['agents', osqueryPolicies], + ['agents', osqueryPolicies, searchValue, perPage], async () => { + let kuery = `(${osqueryPolicies.map((p) => `policy_id:${p}`).join(' or ')})`; + if (searchValue) { + kuery += ` and (local_metadata.host.hostname:/${searchValue}/ or local_metadata.elastic.agent.id:/${searchValue}/)`; + } return await http.get('/api/fleet/agents', { query: { - kuery: osqueryPolicies.map((p) => `policy_id:${p}`).join(' or '), - perPage: 9000, + kuery, + perPage, }, }); }, diff --git a/x-pack/plugins/osquery/public/live_query/form/agents_table_field.tsx b/x-pack/plugins/osquery/public/live_query/form/agents_table_field.tsx index 4bc9262af7613..ccde0fd8305f9 100644 --- a/x-pack/plugins/osquery/public/live_query/form/agents_table_field.tsx +++ b/x-pack/plugins/osquery/public/live_query/form/agents_table_field.tsx @@ -7,10 +7,11 @@ import React, { useCallback } from 'react'; import { FieldHook } from '../../shared_imports'; -import { AgentsTable, AgentsSelection } from '../../agents/agents_table'; +import { AgentsTable } from '../../agents/agents_table'; +import { AgentSelection } from '../../agents/types'; interface AgentsTableFieldProps { - field: FieldHook; + field: FieldHook; } const AgentsTableFieldComponent: React.FC = ({ field }) => { From a89b75671000d6c8431ff150b4f555e1f00f361e Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 17 Apr 2021 15:52:32 +0100 Subject: [PATCH 18/25] skip flaky suite (#97387) --- x-pack/test/api_integration/apis/lens/existing_fields.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/lens/existing_fields.ts b/x-pack/test/api_integration/apis/lens/existing_fields.ts index 88949401f102a..0358786993919 100644 --- a/x-pack/test/api_integration/apis/lens/existing_fields.ts +++ b/x-pack/test/api_integration/apis/lens/existing_fields.ts @@ -160,7 +160,8 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); - describe('existing_fields apis', () => { + // FLAKY: https://github.com/elastic/kibana/issues/97387 + describe.skip('existing_fields apis', () => { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); await esArchiver.loadIfNeeded('visualize/default'); From 3b31d81196799a9ced9acd5f30082b0f7aed1ce7 Mon Sep 17 00:00:00 2001 From: Catherine Liu Date: Sat, 17 Apr 2021 22:29:27 -0700 Subject: [PATCH 19/25] [Dashboard] Makes lens default editor for creating new panels (#96181) * Makes lens default editor in dashboard Added all editors menu to dashboard panel toolbar Fixed toggle on editor menu Removed unnecessary comments Added data test subjects to editor menu buttons Populated editor menu with vis types Removed unused imports Fixed imports Adds showCreateNewMenu prop to AddPanelFlyout Rearranged order of editor menu options Fixed ts errors Added groupnig to embeddable factory Use embeddable state transfer service to redirect to editors Added showGroups to TypeSelectionState Fixed add panel flyout test Fixed data test subjects Fixed factory groupings Removed unused import Fixed page object Added telemtry to dashboard toolbar Added telemtry to editor menu Fix ml embeddable functional tests Fix lens dashboard test Fix empty dashboard test Fixed ts errors Fixed time to visualize security test Fixed empty dashboard test Fixed clickAddNewEmbeddableLink in dashboardAddPanel service Fixed agg based vis functional tests Revert test changes Fixed typo Fix tests Fix more tests Fix ts errors Fixed more tests Fixed toolbar sizes and margins to align with lens Fix tests Fixed callbacks Fixed button prop type New vis modal copy updates Added savedObjectMetaData to log stream embeddable factory Addressed feedback Fixed ts error Fix more tests Fixed ts errors Updated dashboard empty prompt copy Adds tooltip to log stream embeddable factory saved object meta data Made icons monochrome in toolbar Fixed icon colors in dark mode Cleaned up css Fixed ts errors Updated snapshot Fixed map icon color * Added tooltips for ML embeddables * Restored test * Added empty dashboard panel test * Fixed i18n id * Fix dashboard_embedding test * Removed unused service * Fixed i18n error * Added icon and description properties to embeddable factory definition * Fixed ts errors * Fixed expected value Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...public.embeddablefactory.getdescription.md | 17 ++ ...le-public.embeddablefactory.geticontype.md | 17 ++ ...dable-public.embeddablefactory.grouping.md | 13 + ...ins-embeddable-public.embeddablefactory.md | 3 + ...able-public.embeddablefactorydefinition.md | 2 +- ...ns-embeddable-public.openaddpanelflyout.md | 3 +- src/plugins/dashboard/kibana.json | 3 +- .../public/application/_dashboard_app.scss | 15 +- .../public/application/dashboard_router.tsx | 2 + .../dashboard_container_factory.tsx | 2 +- .../dashboard_empty_screen.test.tsx.snap | 2 +- .../listing/dashboard_listing.test.tsx | 2 + .../application/top_nav/dashboard_top_nav.tsx | 127 ++++++--- .../application/top_nav/editor_menu.tsx | 255 ++++++++++++++++++ .../dashboard/public/application/types.ts | 2 + .../dashboard/public/dashboard_strings.ts | 2 +- src/plugins/dashboard/public/plugin.tsx | 2 + .../default_embeddable_factory_provider.ts | 3 + .../lib/embeddables/embeddable_factory.ts | 17 ++ .../embeddable_factory_definition.ts | 3 + .../add_panel/add_panel_flyout.test.tsx | 2 + .../add_panel/add_panel_flyout.tsx | 5 +- .../add_panel/open_add_panel_flyout.tsx | 3 + src/plugins/embeddable/public/public.api.md | 7 +- .../solution_toolbar/items/button.scss | 1 - .../solution_toolbar/items/button.tsx | 10 +- .../solution_toolbar/items/popover.tsx | 10 +- .../items/primary_button.scss | 20 ++ .../solution_toolbar/items/primary_button.tsx | 18 +- .../solution_toolbar/items/quick_group.scss | 13 + .../solution_toolbar/items/quick_group.tsx | 12 +- .../solution_toolbar/solution_toolbar.scss | 15 +- .../solution_toolbar/solution_toolbar.tsx | 9 +- src/plugins/presentation_util/public/index.ts | 1 + .../visualize_embeddable_factory.tsx | 2 +- src/plugins/visualizations/public/index.ts | 2 +- .../public/wizard/dialog_navigation.tsx | 2 +- .../public/wizard/new_vis_modal.tsx | 7 +- .../public/wizard/show_new_vis.tsx | 7 + test/examples/embeddables/adding_children.ts | 23 +- .../dashboard/create_and_add_embeddables.ts | 26 +- .../dashboard/dashboard_unsaved_listing.ts | 8 +- .../apps/dashboard/dashboard_unsaved_state.ts | 4 +- .../dashboard/edit_embeddable_redirects.ts | 8 +- .../apps/dashboard/edit_visualizations.js | 3 +- .../apps/dashboard/empty_dashboard.ts | 9 +- test/functional/apps/dashboard/view_edit.ts | 6 +- .../functional/page_objects/dashboard_page.ts | 10 - .../services/dashboard/add_panel.ts | 30 ++- .../services/dashboard/visualizations.ts | 45 +--- .../new_visualize_flow/dashboard_embedding.ts | 5 - .../log_stream_embeddable_factory.ts | 10 + .../anomaly_charts_embeddable_factory.ts | 17 +- .../anomaly_swimlane_embeddable_factory.ts | 17 +- .../apps/ml_embeddables_in_dashboard.ts | 4 +- .../apps/dashboard/dashboard_lens_by_value.ts | 2 - .../apps/dashboard/dashboard_maps_by_value.ts | 9 +- .../time_to_visualize_security.ts | 7 +- .../functional/apps/dashboard/sync_colors.ts | 2 - .../dashboard_mode/dashboard_empty_screen.js | 7 - x-pack/test/functional/apps/lens/dashboard.ts | 2 - .../test/functional/apps/lens/lens_tagging.ts | 5 +- .../maps/embeddable/embeddable_library.js | 4 +- .../apps/maps/embeddable/save_and_return.js | 6 +- .../anomaly_charts_dashboard_embeddables.ts | 4 +- .../test/functional/page_objects/lens_page.ts | 3 +- 66 files changed, 684 insertions(+), 230 deletions(-) create mode 100644 docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md create mode 100644 docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md create mode 100644 docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.grouping.md create mode 100644 src/plugins/dashboard/public/application/top_nav/editor_menu.tsx create mode 100644 src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.scss diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md new file mode 100644 index 0000000000000..1699351349bf8 --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [EmbeddableFactory](./kibana-plugin-plugins-embeddable-public.embeddablefactory.md) > [getDescription](./kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md) + +## EmbeddableFactory.getDescription() method + +Returns a description about the embeddable. + +Signature: + +```typescript +getDescription(): string; +``` +Returns: + +`string` + diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md new file mode 100644 index 0000000000000..58b987e5630c4 --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [EmbeddableFactory](./kibana-plugin-plugins-embeddable-public.embeddablefactory.md) > [getIconType](./kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md) + +## EmbeddableFactory.getIconType() method + +Returns an EUI Icon type to be displayed in a menu. + +Signature: + +```typescript +getIconType(): string; +``` +Returns: + +`string` + diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.grouping.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.grouping.md new file mode 100644 index 0000000000000..c4dbe739ddfcb --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.grouping.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [EmbeddableFactory](./kibana-plugin-plugins-embeddable-public.embeddablefactory.md) > [grouping](./kibana-plugin-plugins-embeddable-public.embeddablefactory.grouping.md) + +## EmbeddableFactory.grouping property + +Indicates the grouping this factory should appear in a sub-menu. Example, this is used for grouping options in the editors menu in Dashboard for creating new embeddables + +Signature: + +```typescript +readonly grouping?: UiActionsPresentableGrouping; +``` diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.md index b355acd0567a8..8ee60e1f58a2b 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactory.md @@ -16,6 +16,7 @@ export interface EmbeddableFactoryUiActionsPresentableGrouping | Indicates the grouping this factory should appear in a sub-menu. Example, this is used for grouping options in the editors menu in Dashboard for creating new embeddables | | [isContainerType](./kibana-plugin-plugins-embeddable-public.embeddablefactory.iscontainertype.md) | boolean | True if is this factory create embeddables that are Containers. Used in the add panel to conditionally show whether these can be added to another container. It's just not supported right now, but once nested containers are officially supported we can probably get rid of this interface. | | [isEditable](./kibana-plugin-plugins-embeddable-public.embeddablefactory.iseditable.md) | () => Promise<boolean> | Returns whether the current user should be allowed to edit this type of embeddable. Most of the time this should be based off the capabilities service, hence it's async. | | [savedObjectMetaData](./kibana-plugin-plugins-embeddable-public.embeddablefactory.savedobjectmetadata.md) | SavedObjectMetaData<TSavedObjectAttributes> | | @@ -29,6 +30,8 @@ export interface EmbeddableFactoryThis will likely change in future iterations when we improve in place editing capabilities. | | [createFromSavedObject(savedObjectId, input, parent)](./kibana-plugin-plugins-embeddable-public.embeddablefactory.createfromsavedobject.md) | Creates a new embeddable instance based off the saved object id. | | [getDefaultInput(partial)](./kibana-plugin-plugins-embeddable-public.embeddablefactory.getdefaultinput.md) | Can be used to get any default input, to be passed in to during the creation process. Default input will not be stored in a parent container, so any inherited input from a container will trump default input parameters. | +| [getDescription()](./kibana-plugin-plugins-embeddable-public.embeddablefactory.getdescription.md) | Returns a description about the embeddable. | | [getDisplayName()](./kibana-plugin-plugins-embeddable-public.embeddablefactory.getdisplayname.md) | Returns a display name for this type of embeddable. Used in "Create new... " options in the add panel for containers. | | [getExplicitInput()](./kibana-plugin-plugins-embeddable-public.embeddablefactory.getexplicitinput.md) | Can be used to request explicit input from the user, to be passed in to EmbeddableFactory:create. Explicit input is stored on the parent container for this embeddable. It overrides any inherited input passed down from the parent container. | +| [getIconType()](./kibana-plugin-plugins-embeddable-public.embeddablefactory.geticontype.md) | Returns an EUI Icon type to be displayed in a menu. | diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactorydefinition.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactorydefinition.md index 6ecb88e7c017e..dd61272625160 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactorydefinition.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddablefactorydefinition.md @@ -7,5 +7,5 @@ Signature: ```typescript -export declare type EmbeddableFactoryDefinition = IEmbeddable, T extends SavedObjectAttributes = SavedObjectAttributes> = Pick, 'create' | 'type' | 'isEditable' | 'getDisplayName'> & Partial, 'createFromSavedObject' | 'isContainerType' | 'getExplicitInput' | 'savedObjectMetaData' | 'canCreateNew' | 'getDefaultInput' | 'telemetry' | 'extract' | 'inject' | 'migrations'>>; +export declare type EmbeddableFactoryDefinition = IEmbeddable, T extends SavedObjectAttributes = SavedObjectAttributes> = Pick, 'create' | 'type' | 'isEditable' | 'getDisplayName'> & Partial, 'createFromSavedObject' | 'isContainerType' | 'getExplicitInput' | 'savedObjectMetaData' | 'canCreateNew' | 'getDefaultInput' | 'telemetry' | 'extract' | 'inject' | 'migrations' | 'grouping' | 'getIconType' | 'getDescription'>>; ``` diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.openaddpanelflyout.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.openaddpanelflyout.md index add4646375359..90caaa3035b34 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.openaddpanelflyout.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.openaddpanelflyout.md @@ -14,6 +14,7 @@ export declare function openAddPanelFlyout(options: { overlays: OverlayStart; notifications: NotificationsStart; SavedObjectFinder: React.ComponentType; + showCreateNewMenu?: boolean; }): OverlayRef; ``` @@ -21,7 +22,7 @@ export declare function openAddPanelFlyout(options: { | Parameter | Type | Description | | --- | --- | --- | -| options | {
embeddable: IContainer;
getFactory: EmbeddableStart['getEmbeddableFactory'];
getAllFactories: EmbeddableStart['getEmbeddableFactories'];
overlays: OverlayStart;
notifications: NotificationsStart;
SavedObjectFinder: React.ComponentType<any>;
} | | +| options | {
embeddable: IContainer;
getFactory: EmbeddableStart['getEmbeddableFactory'];
getAllFactories: EmbeddableStart['getEmbeddableFactories'];
overlays: OverlayStart;
notifications: NotificationsStart;
SavedObjectFinder: React.ComponentType<any>;
showCreateNewMenu?: boolean;
} | | Returns: diff --git a/src/plugins/dashboard/kibana.json b/src/plugins/dashboard/kibana.json index 41335069461fa..54eaf461b73d7 100644 --- a/src/plugins/dashboard/kibana.json +++ b/src/plugins/dashboard/kibana.json @@ -11,7 +11,8 @@ "share", "uiActions", "urlForwarding", - "presentationUtil" + "presentationUtil", + "visualizations" ], "optionalPlugins": [ "home", diff --git a/src/plugins/dashboard/public/application/_dashboard_app.scss b/src/plugins/dashboard/public/application/_dashboard_app.scss index 30253afff391f..f6525377cce70 100644 --- a/src/plugins/dashboard/public/application/_dashboard_app.scss +++ b/src/plugins/dashboard/public/application/_dashboard_app.scss @@ -66,4 +66,17 @@ .dshUnsavedListingItem__actions { flex-direction: column; } -} \ No newline at end of file +} + +// Temporary fix for two tone icons to make them monochrome +.dshSolutionToolbar__editorContextMenu--dark { + .euiIcon path { + fill: $euiColorGhost; + } +} + +.dshSolutionToolbar__editorContextMenu--light { + .euiIcon path { + fill: $euiColorInk; + } +} diff --git a/src/plugins/dashboard/public/application/dashboard_router.tsx b/src/plugins/dashboard/public/application/dashboard_router.tsx index e5281a257ee13..ed68afc5e97b1 100644 --- a/src/plugins/dashboard/public/application/dashboard_router.tsx +++ b/src/plugins/dashboard/public/application/dashboard_router.tsx @@ -80,6 +80,7 @@ export async function mountApp({ embeddable: embeddableStart, kibanaLegacy: { dashboardConfig }, savedObjectsTaggingOss, + visualizations, } = pluginsStart; const spacesApi = pluginsStart.spacesOss?.isSpacesAvailable ? pluginsStart.spacesOss : undefined; @@ -123,6 +124,7 @@ export async function mountApp({ visualizeCapabilities: { save: Boolean(coreStart.application.capabilities.visualize?.save) }, storeSearchSession: Boolean(coreStart.application.capabilities.dashboard.storeSearchSession), }, + visualizations, }; const getUrlStateStorage = (history: RouteComponentProps['history']) => diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container_factory.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container_factory.tsx index 9b93f0bbd0711..ff592742488f5 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container_factory.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container_factory.tsx @@ -49,7 +49,7 @@ export class DashboardContainerFactoryDefinition public readonly getDisplayName = () => { return i18n.translate('dashboard.factory.displayName', { - defaultMessage: 'dashboard', + defaultMessage: 'Dashboard', }); }; diff --git a/src/plugins/dashboard/public/application/embeddable/empty_screen/__snapshots__/dashboard_empty_screen.test.tsx.snap b/src/plugins/dashboard/public/application/embeddable/empty_screen/__snapshots__/dashboard_empty_screen.test.tsx.snap index 4cd3eb13f3609..138d665866af0 100644 --- a/src/plugins/dashboard/public/application/embeddable/empty_screen/__snapshots__/dashboard_empty_screen.test.tsx.snap +++ b/src/plugins/dashboard/public/application/embeddable/empty_screen/__snapshots__/dashboard_empty_screen.test.tsx.snap @@ -287,7 +287,7 @@ exports[`DashboardEmptyScreen renders correctly with edit mode 1`] = `

- Add your first panel + Add your first visualization

().services; const [state, setState] = useState({ chromeIsVisible: false }); const [isSaveInProgress, setIsSaveInProgress] = useState(false); + const lensAlias = visualizations.getAliases().find(({ name }) => name === 'lens'); + const quickButtonVisTypes = ['markdown', 'maps']; const stateTransferService = embeddable.getStateTransfer(); + const IS_DARK_THEME = uiSettings.get('theme:darkMode'); + + const trackUiMetric = usageCollection?.reportUiCounter.bind( + usageCollection, + DashboardConstants.DASHBOARDS_ID + ); useEffect(() => { const visibleSubscription = chrome.getIsVisible$().subscribe((chromeIsVisible) => { @@ -152,27 +161,36 @@ export function DashboardTopNav({ uiSettings, ]); - const createNew = useCallback(async () => { - const type = 'visualization'; - const factory = embeddable.getEmbeddableFactory(type); + const createNewVisType = useCallback( + (visType?: BaseVisType | VisTypeAlias) => () => { + let path = ''; + let appId = ''; - if (!factory) { - throw new EmbeddableFactoryNotFoundError(type); - } + if (visType) { + if (trackUiMetric) { + trackUiMetric(METRIC_TYPE.CLICK, visType.name); + } - await factory.create({} as EmbeddableInput, dashboardContainer); - }, [dashboardContainer, embeddable]); + if ('aliasPath' in visType) { + appId = visType.aliasApp; + path = visType.aliasPath; + } else { + appId = 'visualize'; + path = `#/create?type=${encodeURIComponent(visType.name)}`; + } + } else { + appId = 'visualize'; + path = '#/create?'; + } - const createNewVisType = useCallback( - (newVisType: string) => async () => { - stateTransferService.navigateToEditor('visualize', { - path: `#/create?type=${encodeURIComponent(newVisType)}`, + stateTransferService.navigateToEditor(appId, { + path, state: { originatingApp: DashboardConstants.DASHBOARDS_ID, }, }); }, - [stateTransferService] + [trackUiMetric, stateTransferService] ); const clearAddPanel = useCallback(() => { @@ -563,38 +581,57 @@ export function DashboardTopNav({ const { TopNavMenu } = navigation.ui; - const quickButtons = [ - { - iconType: 'visText', - createType: i18n.translate('dashboard.solutionToolbar.markdownQuickButtonLabel', { - defaultMessage: 'Markdown', - }), - onClick: createNewVisType('markdown'), - 'data-test-subj': 'dashboardMarkdownQuickButton', - }, - { - iconType: 'controlsHorizontal', - createType: i18n.translate('dashboard.solutionToolbar.inputControlsQuickButtonLabel', { - defaultMessage: 'Input control', - }), - onClick: createNewVisType('input_control_vis'), - 'data-test-subj': 'dashboardInputControlsQuickButton', - }, - ]; + const getVisTypeQuickButton = (visTypeName: string) => { + const visType = + visualizations.get(visTypeName) || + visualizations.getAliases().find(({ name }) => name === visTypeName); + + if (visType) { + if ('aliasPath' in visType) { + const { name, icon, title } = visType as VisTypeAlias; + + return { + iconType: icon, + createType: title, + onClick: createNewVisType(visType as VisTypeAlias), + 'data-test-subj': `dashboardQuickButton${name}`, + isDarkModeEnabled: IS_DARK_THEME, + }; + } else { + const { name, icon, title, titleInWizard } = visType as BaseVisType; + + return { + iconType: icon, + createType: titleInWizard || title, + onClick: createNewVisType(visType as BaseVisType), + 'data-test-subj': `dashboardQuickButton${name}`, + isDarkModeEnabled: IS_DARK_THEME, + }; + } + } + + return; + }; + + const quickButtons = quickButtonVisTypes + .map(getVisTypeQuickButton) + .filter((button) => button) as QuickButtonProps[]; return ( <> + {viewMode !== ViewMode.VIEW ? ( - + {{ primaryActionButton: ( ), @@ -605,6 +642,12 @@ export function DashboardTopNav({ data-test-subj="dashboardAddPanelButton" /> ), + extraButtons: [ + , + ], }} ) : null} diff --git a/src/plugins/dashboard/public/application/top_nav/editor_menu.tsx b/src/plugins/dashboard/public/application/top_nav/editor_menu.tsx new file mode 100644 index 0000000000000..5205f5b294c4f --- /dev/null +++ b/src/plugins/dashboard/public/application/top_nav/editor_menu.tsx @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useCallback } from 'react'; +import { + EuiContextMenu, + EuiContextMenuPanelItemDescriptor, + EuiContextMenuItemIcon, +} from '@elastic/eui'; +import { METRIC_TYPE } from '@kbn/analytics'; +import { i18n } from '@kbn/i18n'; +import { BaseVisType, VisGroups, VisTypeAlias } from '../../../../visualizations/public'; +import { SolutionToolbarPopover } from '../../../../presentation_util/public'; +import { EmbeddableFactoryDefinition, EmbeddableInput } from '../../services/embeddable'; +import { useKibana } from '../../services/kibana_react'; +import { DashboardAppServices } from '../types'; +import { DashboardContainer } from '..'; +import { DashboardConstants } from '../../dashboard_constants'; +import { dashboardReplacePanelAction } from '../../dashboard_strings'; + +interface Props { + /** Dashboard container */ + dashboardContainer: DashboardContainer; + /** Handler for creating new visualization of a specified type */ + createNewVisType: (visType: BaseVisType | VisTypeAlias) => () => void; +} + +interface FactoryGroup { + id: string; + appName: string; + icon: EuiContextMenuItemIcon; + panelId: number; + factories: EmbeddableFactoryDefinition[]; +} + +export const EditorMenu = ({ dashboardContainer, createNewVisType }: Props) => { + const { + core, + embeddable, + visualizations, + usageCollection, + uiSettings, + } = useKibana().services; + + const IS_DARK_THEME = uiSettings.get('theme:darkMode'); + + const trackUiMetric = usageCollection?.reportUiCounter.bind( + usageCollection, + DashboardConstants.DASHBOARDS_ID + ); + + const createNewAggsBasedVis = useCallback( + (visType?: BaseVisType) => () => + visualizations.showNewVisModal({ + originatingApp: DashboardConstants.DASHBOARDS_ID, + outsideVisualizeApp: true, + showAggsSelection: true, + selectedVisType: visType, + }), + [visualizations] + ); + + const getVisTypesByGroup = (group: VisGroups) => + visualizations + .getByGroup(group) + .sort(({ name: a }: BaseVisType | VisTypeAlias, { name: b }: BaseVisType | VisTypeAlias) => { + if (a < b) { + return -1; + } + if (a > b) { + return 1; + } + return 0; + }) + .filter(({ hidden }: BaseVisType) => !hidden); + + const promotedVisTypes = getVisTypesByGroup(VisGroups.PROMOTED); + const aggsBasedVisTypes = getVisTypesByGroup(VisGroups.AGGBASED); + const toolVisTypes = getVisTypesByGroup(VisGroups.TOOLS); + const visTypeAliases = visualizations + .getAliases() + .sort(({ promotion: a = false }: VisTypeAlias, { promotion: b = false }: VisTypeAlias) => + a === b ? 0 : a ? -1 : 1 + ); + + const factories = embeddable + ? Array.from(embeddable.getEmbeddableFactories()).filter( + ({ type, isEditable, canCreateNew, isContainerType }) => + isEditable() && !isContainerType && canCreateNew() && type !== 'visualization' + ) + : []; + + const factoryGroupMap: Record = {}; + const ungroupedFactories: EmbeddableFactoryDefinition[] = []; + const aggBasedPanelID = 1; + + let panelCount = 1 + aggBasedPanelID; + + factories.forEach((factory: EmbeddableFactoryDefinition, index) => { + const { grouping } = factory; + + if (grouping) { + grouping.forEach((group) => { + if (factoryGroupMap[group.id]) { + factoryGroupMap[group.id].factories.push(factory); + } else { + factoryGroupMap[group.id] = { + id: group.id, + appName: group.getDisplayName ? group.getDisplayName({ embeddable }) : group.id, + icon: (group.getIconType + ? group.getIconType({ embeddable }) + : 'empty') as EuiContextMenuItemIcon, + factories: [factory], + panelId: panelCount, + }; + + panelCount++; + } + }); + } else { + ungroupedFactories.push(factory); + } + }); + + const getVisTypeMenuItem = (visType: BaseVisType): EuiContextMenuPanelItemDescriptor => { + const { name, title, titleInWizard, description, icon = 'empty', group } = visType; + return { + name: titleInWizard || title, + icon: icon as string, + onClick: + group === VisGroups.AGGBASED ? createNewAggsBasedVis(visType) : createNewVisType(visType), + 'data-test-subj': `visType-${name}`, + toolTipContent: description, + }; + }; + + const getVisTypeAliasMenuItem = ( + visTypeAlias: VisTypeAlias + ): EuiContextMenuPanelItemDescriptor => { + const { name, title, description, icon = 'empty' } = visTypeAlias; + + return { + name: title, + icon, + onClick: createNewVisType(visTypeAlias), + 'data-test-subj': `visType-${name}`, + toolTipContent: description, + }; + }; + + const getEmbeddableFactoryMenuItem = ( + factory: EmbeddableFactoryDefinition + ): EuiContextMenuPanelItemDescriptor => { + const icon = factory?.getIconType ? factory.getIconType() : 'empty'; + + const toolTipContent = factory?.getDescription ? factory.getDescription() : undefined; + + return { + name: factory.getDisplayName(), + icon, + toolTipContent, + onClick: async () => { + if (trackUiMetric) { + trackUiMetric(METRIC_TYPE.CLICK, factory.type); + } + let newEmbeddable; + if (factory.getExplicitInput) { + const explicitInput = await factory.getExplicitInput(); + newEmbeddable = await dashboardContainer.addNewEmbeddable(factory.type, explicitInput); + } else { + newEmbeddable = await factory.create({} as EmbeddableInput, dashboardContainer); + } + + if (newEmbeddable) { + core.notifications.toasts.addSuccess({ + title: dashboardReplacePanelAction.getSuccessMessage( + `'${newEmbeddable.getInput().title}'` || '' + ), + 'data-test-subj': 'addEmbeddableToDashboardSuccess', + }); + } + }, + 'data-test-subj': `createNew-${factory.type}`, + }; + }; + + const aggsPanelTitle = i18n.translate('dashboard.editorMenu.aggBasedGroupTitle', { + defaultMessage: 'Aggregation based', + }); + + const editorMenuPanels = [ + { + id: 0, + items: [ + ...visTypeAliases.map(getVisTypeAliasMenuItem), + ...Object.values(factoryGroupMap).map(({ id, appName, icon, panelId }) => ({ + name: appName, + icon, + panel: panelId, + 'data-test-subj': `dashboardEditorMenu-${id}Group`, + })), + ...ungroupedFactories.map(getEmbeddableFactoryMenuItem), + ...promotedVisTypes.map(getVisTypeMenuItem), + { + name: aggsPanelTitle, + icon: 'visualizeApp', + panel: aggBasedPanelID, + 'data-test-subj': `dashboardEditorAggBasedMenuItem`, + }, + ...toolVisTypes.map(getVisTypeMenuItem), + ], + }, + { + id: aggBasedPanelID, + title: aggsPanelTitle, + items: aggsBasedVisTypes.map(getVisTypeMenuItem), + }, + ...Object.values(factoryGroupMap).map( + ({ appName, panelId, factories: groupFactories }: FactoryGroup) => ({ + id: panelId, + title: appName, + items: groupFactories.map(getEmbeddableFactoryMenuItem), + }) + ), + ]; + + return ( + + + + ); +}; diff --git a/src/plugins/dashboard/public/application/types.ts b/src/plugins/dashboard/public/application/types.ts index 6415fdfd73ee8..dd291291ce9d6 100644 --- a/src/plugins/dashboard/public/application/types.ts +++ b/src/plugins/dashboard/public/application/types.ts @@ -25,6 +25,7 @@ import { DataPublicPluginStart, IndexPatternsContract } from '../services/data'; import { SavedObjectLoader, SavedObjectsStart } from '../services/saved_objects'; import { DashboardPanelStorage } from './lib'; import { UrlForwardingStart } from '../../../url_forwarding/public'; +import { VisualizationsStart } from '../../../visualizations/public'; export type DashboardRedirect = (props: RedirectToProps) => void; export type RedirectToProps = @@ -83,4 +84,5 @@ export interface DashboardAppServices { savedObjectsClient: SavedObjectsClientContract; setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; savedQueryService: DataPublicPluginStart['query']['savedQueries']; + visualizations: VisualizationsStart; } diff --git a/src/plugins/dashboard/public/dashboard_strings.ts b/src/plugins/dashboard/public/dashboard_strings.ts index 79a59d0cfa605..531ff815312cf 100644 --- a/src/plugins/dashboard/public/dashboard_strings.ts +++ b/src/plugins/dashboard/public/dashboard_strings.ts @@ -377,7 +377,7 @@ export const emptyScreenStrings = { }), getEmptyWidgetTitle: () => i18n.translate('dashboard.emptyWidget.addPanelTitle', { - defaultMessage: 'Add your first panel', + defaultMessage: 'Add your first visualization', }), getEmptyWidgetDescription: () => i18n.translate('dashboard.emptyWidget.addPanelDescription', { diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index e2f52a47455b3..0fad1c51f433a 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -24,6 +24,7 @@ import { PluginInitializerContext, SavedObjectsClientContract, } from '../../../core/public'; +import { VisualizationsStart } from '../../visualizations/public'; import { createKbnUrlTracker } from './services/kibana_utils'; import { UsageCollectionSetup } from './services/usage_collection'; @@ -115,6 +116,7 @@ export interface DashboardStartDependencies { presentationUtil: PresentationUtilPluginStart; savedObjectsTaggingOss?: SavedObjectTaggingOssPluginStart; spacesOss?: SpacesOssPluginStart; + visualizations: VisualizationsStart; } export type DashboardSetup = void; diff --git a/src/plugins/embeddable/public/lib/embeddables/default_embeddable_factory_provider.ts b/src/plugins/embeddable/public/lib/embeddables/default_embeddable_factory_provider.ts index 27164b3cddbc2..b260c594591fa 100644 --- a/src/plugins/embeddable/public/lib/embeddables/default_embeddable_factory_provider.ts +++ b/src/plugins/embeddable/public/lib/embeddables/default_embeddable_factory_provider.ts @@ -37,11 +37,14 @@ export const defaultEmbeddableFactoryProvider = < type: def.type, isEditable: def.isEditable.bind(def), getDisplayName: def.getDisplayName.bind(def), + getDescription: def.getDescription ? def.getDescription.bind(def) : () => '', + getIconType: def.getIconType ? def.getIconType.bind(def) : () => 'empty', savedObjectMetaData: def.savedObjectMetaData, telemetry: def.telemetry || (() => ({})), inject: def.inject || ((state: EmbeddableStateWithType) => state), extract: def.extract || ((state: EmbeddableStateWithType) => ({ state, references: [] })), migrations: def.migrations || {}, + grouping: def.grouping, }; return factory; }; diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable_factory.ts b/src/plugins/embeddable/public/lib/embeddables/embeddable_factory.ts index 7f3277130f90f..6ec035f442dd2 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable_factory.ts +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable_factory.ts @@ -14,6 +14,7 @@ import { IContainer } from '../containers/i_container'; import { PropertySpec } from '../types'; import { PersistableState } from '../../../../kibana_utils/common'; import { EmbeddableStateWithType } from '../../../common/types'; +import { UiActionsPresentableGrouping } from '../../../../ui_actions/public'; export interface EmbeddableInstanceConfiguration { id: string; @@ -48,6 +49,12 @@ export interface EmbeddableFactory< readonly savedObjectMetaData?: SavedObjectMetaData; + /** + * Indicates the grouping this factory should appear in a sub-menu. Example, this is used for grouping + * options in the editors menu in Dashboard for creating new embeddables + */ + readonly grouping?: UiActionsPresentableGrouping; + /** * True if is this factory create embeddables that are Containers. Used in the add panel to * conditionally show whether these can be added to another container. It's just not @@ -62,6 +69,16 @@ export interface EmbeddableFactory< */ getDisplayName(): string; + /** + * Returns an EUI Icon type to be displayed in a menu. + */ + getIconType(): string; + + /** + * Returns a description about the embeddable. + */ + getDescription(): string; + /** * If false, this type of embeddable can't be created with the "createNew" functionality. Instead, * use createFromSavedObject, where an existing saved object must first exist. diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable_factory_definition.ts b/src/plugins/embeddable/public/lib/embeddables/embeddable_factory_definition.ts index a64aa32c6e7c4..f2819f2a2e664 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable_factory_definition.ts +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable_factory_definition.ts @@ -33,5 +33,8 @@ export type EmbeddableFactoryDefinition< | 'extract' | 'inject' | 'migrations' + | 'grouping' + | 'getIconType' + | 'getDescription' > >; diff --git a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.test.tsx b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.test.tsx index 432897763aa04..1c96945f014c8 100644 --- a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.test.tsx +++ b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.test.tsx @@ -61,6 +61,7 @@ test('createNewEmbeddable() add embeddable to container', async () => { getAllFactories={start.getEmbeddableFactories} notifications={core.notifications} SavedObjectFinder={() => null} + showCreateNewMenu /> ) as ReactWrapper; @@ -112,6 +113,7 @@ test('selecting embeddable in "Create new ..." list calls createNewEmbeddable()' getAllFactories={start.getEmbeddableFactories} notifications={core.notifications} SavedObjectFinder={(props) => } + showCreateNewMenu /> ) as ReactWrapper; diff --git a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.tsx b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.tsx index 8caec4a4428c3..6d6a68d7e5e2a 100644 --- a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.tsx +++ b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/add_panel_flyout.tsx @@ -26,6 +26,7 @@ interface Props { getAllFactories: EmbeddableStart['getEmbeddableFactories']; notifications: CoreSetup['notifications']; SavedObjectFinder: React.ComponentType; + showCreateNewMenu?: boolean; } interface State { @@ -134,7 +135,9 @@ export class AddPanelFlyout extends React.Component { defaultMessage: 'No matching objects found.', })} > - + {this.props.showCreateNewMenu ? ( + + ) : null} ); diff --git a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/open_add_panel_flyout.tsx b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/open_add_panel_flyout.tsx index bed97c82095c7..f0c6e81644b3d 100644 --- a/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/open_add_panel_flyout.tsx +++ b/src/plugins/embeddable/public/lib/panel/panel_header/panel_actions/add_panel/open_add_panel_flyout.tsx @@ -20,6 +20,7 @@ export function openAddPanelFlyout(options: { overlays: OverlayStart; notifications: NotificationsStart; SavedObjectFinder: React.ComponentType; + showCreateNewMenu?: boolean; }): OverlayRef { const { embeddable, @@ -28,6 +29,7 @@ export function openAddPanelFlyout(options: { overlays, notifications, SavedObjectFinder, + showCreateNewMenu, } = options; const flyoutSession = overlays.openFlyout( toMountPoint( @@ -42,6 +44,7 @@ export function openAddPanelFlyout(options: { getAllFactories={getAllFactories} notifications={notifications} SavedObjectFinder={SavedObjectFinder} + showCreateNewMenu={showCreateNewMenu} /> ), { diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md index 220039de2f34e..d522a4e5fa8e8 100644 --- a/src/plugins/embeddable/public/public.api.md +++ b/src/plugins/embeddable/public/public.api.md @@ -378,8 +378,12 @@ export interface EmbeddableFactory; createFromSavedObject(savedObjectId: string, input: Partial, parent?: IContainer): Promise; getDefaultInput(partial: Partial): Partial; + getDescription(): string; getDisplayName(): string; getExplicitInput(): Promise>; + getIconType(): string; + // Warning: (ae-forgotten-export) The symbol "PresentableGrouping" needs to be exported by the entry point index.d.ts + readonly grouping?: PresentableGrouping; readonly isContainerType: boolean; readonly isEditable: () => Promise; // Warning: (ae-forgotten-export) The symbol "SavedObjectMetaData" needs to be exported by the entry point index.d.ts @@ -393,7 +397,7 @@ export interface EmbeddableFactory = IEmbeddable, T extends SavedObjectAttributes = SavedObjectAttributes> = Pick, 'create' | 'type' | 'isEditable' | 'getDisplayName'> & Partial, 'createFromSavedObject' | 'isContainerType' | 'getExplicitInput' | 'savedObjectMetaData' | 'canCreateNew' | 'getDefaultInput' | 'telemetry' | 'extract' | 'inject' | 'migrations'>>; +export type EmbeddableFactoryDefinition = IEmbeddable, T extends SavedObjectAttributes = SavedObjectAttributes> = Pick, 'create' | 'type' | 'isEditable' | 'getDisplayName'> & Partial, 'createFromSavedObject' | 'isContainerType' | 'getExplicitInput' | 'savedObjectMetaData' | 'canCreateNew' | 'getDefaultInput' | 'telemetry' | 'extract' | 'inject' | 'migrations' | 'grouping' | 'getIconType' | 'getDescription'>>; // Warning: (ae-missing-release-tag) "EmbeddableFactoryNotFoundError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -724,6 +728,7 @@ export function openAddPanelFlyout(options: { overlays: OverlayStart_2; notifications: NotificationsStart_2; SavedObjectFinder: React.ComponentType; + showCreateNewMenu?: boolean; }): OverlayRef_2; // Warning: (ae-missing-release-tag) "OutputSpec" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/button.scss b/src/plugins/presentation_util/public/components/solution_toolbar/items/button.scss index 79c3d4cca7ace..b8022201acf59 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/button.scss +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/button.scss @@ -1,4 +1,3 @@ - .solutionToolbarButton { line-height: $euiButtonHeight; // Keeps alignment of text and chart icon background-color: $euiColorEmptyShade; diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/button.tsx b/src/plugins/presentation_util/public/components/solution_toolbar/items/button.tsx index 5de8e24ef5f0d..ee1bbd64b5f87 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/button.tsx +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/button.tsx @@ -12,17 +12,19 @@ import { EuiButtonPropsForButton } from '@elastic/eui/src/components/button/butt import './button.scss'; -export interface Props extends Pick { +export interface Props + extends Pick { label: string; primary?: boolean; + isDarkModeEnabled?: boolean; } -export const SolutionToolbarButton = ({ label, primary, ...rest }: Props) => ( +export const SolutionToolbarButton = ({ label, primary, className, ...rest }: Props) => ( {label} diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/popover.tsx b/src/plugins/presentation_util/public/components/solution_toolbar/items/popover.tsx index fbb34e165190d..33850005b498b 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/popover.tsx +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/popover.tsx @@ -20,14 +20,20 @@ type AllowedPopoverProps = Omit< export type Props = AllowedButtonProps & AllowedPopoverProps; -export const SolutionToolbarPopover = ({ label, iconType, primary, ...popover }: Props) => { +export const SolutionToolbarPopover = ({ + label, + iconType, + primary, + iconSide, + ...popover +}: Props) => { const [isOpen, setIsOpen] = useState(false); const onButtonClick = () => setIsOpen((status) => !status); const closePopover = () => setIsOpen(false); const button = ( - + ); return ( diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.scss b/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.scss new file mode 100644 index 0000000000000..c3d89f430d70c --- /dev/null +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.scss @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// Temporary fix for lensApp icon not support ghost color +.solutionToolbar__primaryButton--dark { + .euiIcon path { + fill: $euiColorInk; + } +} + +.solutionToolbar__primaryButton--light { + .euiIcon path { + fill: $euiColorGhost; + } +} diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.tsx b/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.tsx index e2ef75e45a404..dcf16228ac63b 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.tsx +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/primary_button.tsx @@ -10,6 +10,20 @@ import React from 'react'; import { SolutionToolbarButton, Props as SolutionToolbarButtonProps } from './button'; -export const PrimaryActionButton = (props: Omit) => ( - +import './primary_button.scss'; + +export interface Props extends Omit { + isDarkModeEnabled?: boolean; +} + +export const PrimaryActionButton = ({ isDarkModeEnabled, ...props }: Props) => ( + ); diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.scss b/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.scss index 639ff5bf2a117..870a9a945ed5d 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.scss +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.scss @@ -2,4 +2,17 @@ .quickButtonGroup__button { background-color: $euiColorEmptyShade; } + + // Temporary fix for two tone icons to make them monochrome + .quickButtonGroup__button--dark { + .euiIcon path { + fill: $euiColorGhost; + } + } + // Temporary fix for two tone icons to make them monochrome + .quickButtonGroup__button--light { + .euiIcon path { + fill: $euiColorInk; + } + } } diff --git a/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.tsx b/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.tsx index 58f8bd803b636..eb0a395548cd9 100644 --- a/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.tsx +++ b/src/plugins/presentation_util/public/components/solution_toolbar/items/quick_group.tsx @@ -17,23 +17,27 @@ import './quick_group.scss'; export interface QuickButtonProps extends Pick { createType: string; onClick: () => void; + isDarkModeEnabled?: boolean; } export interface Props { buttons: QuickButtonProps[]; } -type Option = EuiButtonGroupOptionProps & Omit; +type Option = EuiButtonGroupOptionProps & + Omit; export const QuickButtonGroup = ({ buttons }: Props) => { const buttonGroupOptions: Option[] = buttons.map((button: QuickButtonProps, index) => { - const { createType: label, ...rest } = button; + const { createType: label, isDarkModeEnabled, ...rest } = button; const title = strings.getAriaButtonLabel(label); return { ...rest, 'aria-label': title, - className: 'quickButtonGroup__button', + className: `quickButtonGroup__button ${ + isDarkModeEnabled ? 'quickButtonGroup__button--dark' : 'quickButtonGroup__button--light' + }`, id: `${htmlIdGenerator()()}${index}`, label, title, @@ -46,7 +50,7 @@ export const QuickButtonGroup = ({ buttons }: Props) => { return ( { +export const SolutionToolbar = ({ isDarkModeEnabled, children }: Props) => { const { primaryActionButton, quickButtonGroup, @@ -49,8 +50,10 @@ export const SolutionToolbar = ({ children }: Props) => { return ( {primaryActionButton} diff --git a/src/plugins/presentation_util/public/index.ts b/src/plugins/presentation_util/public/index.ts index 9c5f65de40955..fd3ae89419297 100644 --- a/src/plugins/presentation_util/public/index.ts +++ b/src/plugins/presentation_util/public/index.ts @@ -19,6 +19,7 @@ export { LazySavedObjectSaveModalDashboard, withSuspense, } from './components'; + export { AddFromLibraryButton, PrimaryActionButton, diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx b/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx index 2b5a611cd946e..48bff8d203ebd 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable_factory.tsx @@ -113,7 +113,7 @@ export class VisualizeEmbeddableFactory public getDisplayName() { return i18n.translate('visualizations.displayName', { - defaultMessage: 'visualization', + defaultMessage: 'Visualization', }); } diff --git a/src/plugins/visualizations/public/index.ts b/src/plugins/visualizations/public/index.ts index e5b1ba73d9d1c..dbcbb864d2316 100644 --- a/src/plugins/visualizations/public/index.ts +++ b/src/plugins/visualizations/public/index.ts @@ -25,7 +25,7 @@ export { getVisSchemas } from './vis_schemas'; /** @public types */ export { VisualizationsSetup, VisualizationsStart }; export { VisGroups } from './vis_types'; -export type { VisTypeAlias, VisTypeDefinition, Schema, ISchemas } from './vis_types'; +export type { BaseVisType, VisTypeAlias, VisTypeDefinition, Schema, ISchemas } from './vis_types'; export { SerializedVis, SerializedVisData, VisData } from './vis'; export type VisualizeEmbeddableFactoryContract = PublicContract; export type VisualizeEmbeddableContract = PublicContract; diff --git a/src/plugins/visualizations/public/wizard/dialog_navigation.tsx b/src/plugins/visualizations/public/wizard/dialog_navigation.tsx index 1de177e12f40d..c92514d54166f 100644 --- a/src/plugins/visualizations/public/wizard/dialog_navigation.tsx +++ b/src/plugins/visualizations/public/wizard/dialog_navigation.tsx @@ -24,7 +24,7 @@ function DialogNavigation(props: DialogNavigationProps) { {i18n.translate('visualizations.newVisWizard.goBackLink', { - defaultMessage: 'Go back', + defaultMessage: 'Select a different visualization', })} diff --git a/src/plugins/visualizations/public/wizard/new_vis_modal.tsx b/src/plugins/visualizations/public/wizard/new_vis_modal.tsx index d36b734f75be2..317f9d1bb363d 100644 --- a/src/plugins/visualizations/public/wizard/new_vis_modal.tsx +++ b/src/plugins/visualizations/public/wizard/new_vis_modal.tsx @@ -41,6 +41,8 @@ interface TypeSelectionProps { outsideVisualizeApp?: boolean; stateTransfer?: EmbeddableStateTransfer; originatingApp?: string; + showAggsSelection?: boolean; + selectedVisType?: BaseVisType; } interface TypeSelectionState { @@ -69,8 +71,9 @@ class NewVisModal extends React.Component import('./new_vis_modal')); @@ -29,6 +30,8 @@ export interface ShowNewVisModalParams { originatingApp?: string; outsideVisualizeApp?: boolean; createByValue?: boolean; + showAggsSelection?: boolean; + selectedVisType?: BaseVisType; } /** @@ -41,6 +44,8 @@ export function showNewVisModal({ onClose, originatingApp, outsideVisualizeApp, + showAggsSelection, + selectedVisType, }: ShowNewVisModalParams = {}) { const container = document.createElement('div'); let isClosed = false; @@ -78,6 +83,8 @@ export function showNewVisModal({ usageCollection={getUsageCollector()} application={getApplication()} docLinks={getDocLinks()} + showAggsSelection={showAggsSelection} + selectedVisType={selectedVisType} /> diff --git a/test/examples/embeddables/adding_children.ts b/test/examples/embeddables/adding_children.ts index 8b59012bf9825..ee06622a33f51 100644 --- a/test/examples/embeddables/adding_children.ts +++ b/test/examples/embeddables/adding_children.ts @@ -13,31 +13,12 @@ import { PluginFunctionalProviderContext } from 'test/plugin_functional/services export default function ({ getService }: PluginFunctionalProviderContext) { const testSubjects = getService('testSubjects'); const flyout = getService('flyout'); - const retry = getService('retry'); - describe('creating and adding children', () => { + describe('adding children', () => { before(async () => { await testSubjects.click('embeddablePanelExample'); }); - it('Can create a new child', async () => { - await testSubjects.click('embeddablePanelToggleMenuIcon'); - await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL'); - - // this seem like an overkill, but clicking this button which opens context menu was flaky - await testSubjects.waitForEnabled('createNew'); - await retry.waitFor('createNew popover opened', async () => { - await testSubjects.click('createNew'); - return await testSubjects.exists('createNew-TODO_EMBEDDABLE'); - }); - await testSubjects.click('createNew-TODO_EMBEDDABLE'); - - await testSubjects.setValue('taskInputField', 'new task'); - await testSubjects.click('createTodoEmbeddable'); - const tasks = await testSubjects.getVisibleTextAll('todoEmbeddableTask'); - expect(tasks).to.eql(['Goes out on Wednesdays!', 'new task']); - }); - it('Can add a child backed off a saved object', async () => { await testSubjects.click('embeddablePanelToggleMenuIcon'); await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL'); @@ -46,7 +27,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { await testSubjects.moveMouseTo('euiFlyoutCloseButton'); await flyout.ensureClosed('dashboardAddPanel'); const tasks = await testSubjects.getVisibleTextAll('todoEmbeddableTask'); - expect(tasks).to.eql(['Goes out on Wednesdays!', 'new task', 'Take the garbage out']); + expect(tasks).to.eql(['Goes out on Wednesdays!', 'Take the garbage out']); }); }); } diff --git a/test/functional/apps/dashboard/create_and_add_embeddables.ts b/test/functional/apps/dashboard/create_and_add_embeddables.ts index 9b8fc4785a671..3de3b2f843f55 100644 --- a/test/functional/apps/dashboard/create_and_add_embeddables.ts +++ b/test/functional/apps/dashboard/create_and_add_embeddables.ts @@ -35,8 +35,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adds new visualization via the top nav link', async () => { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); await PageObjects.dashboard.switchToEditMode(); - await dashboardAddPanel.clickCreateNewLink(); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationExpectSuccess( @@ -52,9 +52,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adds a new visualization', async () => { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); - await dashboardAddPanel.ensureAddPanelIsShowing(); - await dashboardAddPanel.clickAddNewEmbeddableLink('visualization'); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationExpectSuccess( @@ -71,7 +70,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adds a markdown visualization via the quick button', async () => { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); - await PageObjects.dashboard.clickMarkdownQuickButton(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visualize.saveVisualizationExpectSuccess( 'visualization from markdown quick button', { redirectToOrigin: true } @@ -84,21 +83,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.dashboard.waitForRenderComplete(); }); - it('adds an input control visualization via the quick button', async () => { - const originalPanelCount = await PageObjects.dashboard.getPanelCount(); - await PageObjects.dashboard.clickInputControlsQuickButton(); - await PageObjects.visualize.saveVisualizationExpectSuccess( - 'visualization from input control quick button', - { redirectToOrigin: true } - ); - - await retry.try(async () => { - const panelCount = await PageObjects.dashboard.getPanelCount(); - expect(panelCount).to.eql(originalPanelCount + 1); - }); - await PageObjects.dashboard.waitForRenderComplete(); - }); - it('saves the listing page instead of the visualization to the app link', async () => { await PageObjects.header.clickVisualize(true); const currentUrl = await browser.getCurrentUrl(); diff --git a/test/functional/apps/dashboard/dashboard_unsaved_listing.ts b/test/functional/apps/dashboard/dashboard_unsaved_listing.ts index 233d2e91467fe..1cdc4bbff2c53 100644 --- a/test/functional/apps/dashboard/dashboard_unsaved_listing.ts +++ b/test/functional/apps/dashboard/dashboard_unsaved_listing.ts @@ -25,8 +25,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('dashboard unsaved listing', () => { const addSomePanels = async () => { // add an area chart by value - await dashboardAddPanel.clickCreateNewLink(); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationAndReturn(); @@ -132,8 +132,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.dashboard.switchToEditMode(); // add another panel so we can delete it later - await dashboardAddPanel.clickCreateNewLink(); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationExpectSuccess('Wildvis', { diff --git a/test/functional/apps/dashboard/dashboard_unsaved_state.ts b/test/functional/apps/dashboard/dashboard_unsaved_state.ts index e6cc91880010a..fd203cd8c1356 100644 --- a/test/functional/apps/dashboard/dashboard_unsaved_state.ts +++ b/test/functional/apps/dashboard/dashboard_unsaved_state.ts @@ -41,8 +41,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows the unsaved changes badge after adding panels', async () => { await PageObjects.dashboard.switchToEditMode(); // add an area chart by value - await dashboardAddPanel.clickCreateNewLink(); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationAndReturn(); diff --git a/test/functional/apps/dashboard/edit_embeddable_redirects.ts b/test/functional/apps/dashboard/edit_embeddable_redirects.ts index 8b7b98a59aa12..be540e18a503f 100644 --- a/test/functional/apps/dashboard/edit_embeddable_redirects.ts +++ b/test/functional/apps/dashboard/edit_embeddable_redirects.ts @@ -13,10 +13,9 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['dashboard', 'header', 'visualize', 'settings', 'common']); const esArchiver = getService('esArchiver'); - const testSubjects = getService('testSubjects'); const kibanaServer = getService('kibanaServer'); const dashboardPanelActions = getService('dashboardPanelActions'); - const dashboardVisualizations = getService('dashboardVisualizations'); + const dashboardAddPanel = getService('dashboardAddPanel'); describe('edit embeddable redirects', () => { before(async () => { @@ -88,10 +87,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const newTitle = 'test create panel originatingApp'; await PageObjects.dashboard.loadSavedDashboard('few panels'); await PageObjects.dashboard.switchToEditMode(); - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickMarkdownWidget(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visualize.saveVisualizationExpectSuccess(newTitle, { saveAsNew: true, redirectToOrigin: false, diff --git a/test/functional/apps/dashboard/edit_visualizations.js b/test/functional/apps/dashboard/edit_visualizations.js index ce32f53587e74..b2f21aefcf79c 100644 --- a/test/functional/apps/dashboard/edit_visualizations.js +++ b/test/functional/apps/dashboard/edit_visualizations.js @@ -14,13 +14,14 @@ export default function ({ getService, getPageObjects }) { const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); const kibanaServer = getService('kibanaServer'); + const dashboardAddPanel = getService('dashboardAddPanel'); const dashboardPanelActions = getService('dashboardPanelActions'); const originalMarkdownText = 'Original markdown text'; const modifiedMarkdownText = 'Modified markdown text'; const createMarkdownVis = async (title) => { - await PageObjects.dashboard.clickMarkdownQuickButton(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visEditor.setMarkdownTxt(originalMarkdownText); await PageObjects.visEditor.clickGo(); if (title) { diff --git a/test/functional/apps/dashboard/empty_dashboard.ts b/test/functional/apps/dashboard/empty_dashboard.ts index c096d90aa3595..2cfa6d73dcb72 100644 --- a/test/functional/apps/dashboard/empty_dashboard.ts +++ b/test/functional/apps/dashboard/empty_dashboard.ts @@ -41,15 +41,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should open add panel when add button is clicked', async () => { - await testSubjects.click('dashboardAddPanelButton'); + await dashboardAddPanel.clickOpenAddPanel(); const isAddPanelOpen = await dashboardAddPanel.isAddPanelOpen(); expect(isAddPanelOpen).to.be(true); await testSubjects.click('euiFlyoutCloseButton'); }); it('should add new visualization from dashboard', async () => { - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); await dashboardVisualizations.createAndAddMarkdown({ name: 'Dashboard Test Markdown', markdown: 'Markdown text', @@ -57,5 +55,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.dashboard.waitForRenderComplete(); await dashboardExpect.markdownWithValuesExists(['Markdown text']); }); + + it('should open editor menu when editor button is clicked', async () => { + await dashboardAddPanel.clickEditorMenuButton(); + await testSubjects.existOrFail('dashboardEditorContextMenu'); + }); }); } diff --git a/test/functional/apps/dashboard/view_edit.ts b/test/functional/apps/dashboard/view_edit.ts index c5c7daab27ff1..99a78ebd069c5 100644 --- a/test/functional/apps/dashboard/view_edit.ts +++ b/test/functional/apps/dashboard/view_edit.ts @@ -113,10 +113,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('when a new vis is added', async function () { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); - - await dashboardAddPanel.ensureAddPanelIsShowing(); - await dashboardAddPanel.clickAddNewEmbeddableLink('visualization'); - await PageObjects.visualize.clickAggBasedVisualizations(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); await PageObjects.visualize.clickAreaChart(); await PageObjects.visualize.clickNewSearch(); await PageObjects.visualize.saveVisualizationExpectSuccess('new viz panel', { diff --git a/test/functional/page_objects/dashboard_page.ts b/test/functional/page_objects/dashboard_page.ts index 34559afdf6ae1..9c12296db138c 100644 --- a/test/functional/page_objects/dashboard_page.ts +++ b/test/functional/page_objects/dashboard_page.ts @@ -413,16 +413,6 @@ export function DashboardPageProvider({ getService, getPageObjects }: FtrProvide await testSubjects.click('confirmSaveSavedObjectButton'); } - public async clickMarkdownQuickButton() { - log.debug('Click markdown quick button'); - await testSubjects.click('dashboardMarkdownQuickButton'); - } - - public async clickInputControlsQuickButton() { - log.debug('Click input controls quick button'); - await testSubjects.click('dashboardInputControlsQuickButton'); - } - /** * * @param dashboardTitle {String} diff --git a/test/functional/services/dashboard/add_panel.ts b/test/functional/services/dashboard/add_panel.ts index 7bb1603e0193f..a4e0c8b2647dd 100644 --- a/test/functional/services/dashboard/add_panel.ts +++ b/test/functional/services/dashboard/add_panel.ts @@ -30,15 +30,41 @@ export function DashboardAddPanelProvider({ getService, getPageObjects }: FtrPro await PageObjects.common.sleep(500); } + async clickQuickButton(visType: string) { + log.debug(`DashboardAddPanel.clickQuickButton${visType}`); + await testSubjects.click(`dashboardQuickButton${visType}`); + } + + async clickMarkdownQuickButton() { + await this.clickQuickButton('markdown'); + } + + async clickMapQuickButton() { + await this.clickQuickButton('map'); + } + + async clickEditorMenuButton() { + log.debug('DashboardAddPanel.clickEditorMenuButton'); + await testSubjects.click('dashboardEditorMenuButton'); + } + + async clickAggBasedVisualizations() { + log.debug('DashboardAddPanel.clickEditorMenuAggBasedMenuItem'); + await testSubjects.click('dashboardEditorAggBasedMenuItem'); + } + async clickVisType(visType: string) { log.debug('DashboardAddPanel.clickVisType'); await testSubjects.click(`visType-${visType}`); } + async clickEmbeddableFactoryGroupButton(groupId: string) { + log.debug('DashboardAddPanel.clickEmbeddableFactoryGroupButton'); + await testSubjects.click(`dashboardEditorMenu-${groupId}Group`); + } + async clickAddNewEmbeddableLink(type: string) { - await testSubjects.click('createNew'); await testSubjects.click(`createNew-${type}`); - await testSubjects.missingOrFail(`createNew-${type}`); } async toggleFilterPopover() { diff --git a/test/functional/services/dashboard/visualizations.ts b/test/functional/services/dashboard/visualizations.ts index d1aaa6aa1bd70..2bf7458ff9c5f 100644 --- a/test/functional/services/dashboard/visualizations.ts +++ b/test/functional/services/dashboard/visualizations.ts @@ -10,8 +10,6 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export function DashboardVisualizationProvider({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); - const find = getService('find'); - const retry = getService('retry'); const queryBar = getService('queryBar'); const testSubjects = getService('testSubjects'); const dashboardAddPanel = getService('dashboardAddPanel'); @@ -31,8 +29,8 @@ export function DashboardVisualizationProvider({ getService, getPageObjects }: F if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await dashboardAddPanel.ensureAddPanelIsShowing(); - await dashboardAddPanel.clickAddNewEmbeddableLink('visualization'); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAddNewEmbeddableLink('metrics'); await PageObjects.visualize.clickVisualBuilder(); await PageObjects.visualize.saveVisualizationExpectSuccess(name); } @@ -87,39 +85,13 @@ export function DashboardVisualizationProvider({ getService, getPageObjects }: F await dashboardAddPanel.addSavedSearch(name); } - async clickAddVisualizationButton() { - log.debug('DashboardVisualizations.clickAddVisualizationButton'); - await testSubjects.click('dashboardAddNewPanelButton'); - } - - async isNewVisDialogShowing() { - log.debug('DashboardVisualizations.isNewVisDialogShowing'); - return await find.existsByCssSelector('.visNewVisDialog'); - } - - async ensureNewVisualizationDialogIsShowing() { - let isShowing = await this.isNewVisDialogShowing(); - log.debug(`DashboardVisualizations.ensureNewVisualizationDialogIsShowing:${isShowing}`); - if (!isShowing) { - await retry.try(async () => { - await this.clickAddVisualizationButton(); - isShowing = await this.isNewVisDialogShowing(); - log.debug(`DashboardVisualizations.ensureNewVisualizationDialogIsShowing:${isShowing}`); - if (!isShowing) { - throw new Error('New Vis Dialog still not open, trying again.'); - } - }); - } - } - async createAndAddMarkdown({ name, markdown }: { name: string; markdown: string }) { log.debug(`createAndAddMarkdown(${markdown})`); const inViewMode = await PageObjects.dashboard.getIsInViewMode(); if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await this.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickMarkdownWidget(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visEditor.setMarkdownTxt(markdown); await PageObjects.visEditor.clickGo(); await PageObjects.visualize.saveVisualizationExpectSuccess(name, { @@ -134,10 +106,10 @@ export function DashboardVisualizationProvider({ getService, getPageObjects }: F if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await this.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickAggBasedVisualizations(); - await PageObjects.visualize.clickMetric(); - await find.clickByCssSelector('li.euiListGroupItem:nth-of-type(2)'); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickAggBasedVisualizations(); + await dashboardAddPanel.clickVisType('metric'); + await testSubjects.click('savedObjectTitlelogstash-*'); await testSubjects.exists('visualizesaveAndReturnButton'); await testSubjects.click('visualizesaveAndReturnButton'); } @@ -148,8 +120,7 @@ export function DashboardVisualizationProvider({ getService, getPageObjects }: F if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await this.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickMarkdownWidget(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visEditor.setMarkdownTxt(markdown); await PageObjects.visEditor.clickGo(); await testSubjects.click('visualizesaveAndReturnButton'); diff --git a/test/new_visualize_flow/dashboard_embedding.ts b/test/new_visualize_flow/dashboard_embedding.ts index 6a1315dbfc91e..04b91542223ba 100644 --- a/test/new_visualize_flow/dashboard_embedding.ts +++ b/test/new_visualize_flow/dashboard_embedding.ts @@ -22,7 +22,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); const dashboardExpect = getService('dashboardExpect'); - const testSubjects = getService('testSubjects'); const dashboardVisualizations = getService('dashboardVisualizations'); const PageObjects = getPageObjects([ 'common', @@ -47,8 +46,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adding a metric visualization', async function () { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); expect(originalPanelCount).to.eql(0); - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); await dashboardVisualizations.createAndEmbedMetric('Embedding Vis Test'); await PageObjects.dashboard.waitForRenderComplete(); await dashboardExpect.metricValuesExist(['0']); @@ -59,8 +56,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adding a markdown', async function () { const originalPanelCount = await PageObjects.dashboard.getPanelCount(); expect(originalPanelCount).to.eql(1); - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); await dashboardVisualizations.createAndEmbedMarkdown({ name: 'Embedding Markdown Test', markdown: 'Nice to meet you, markdown is my name', diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts index 4b9b2f99215b7..1c7e8ceb28fb4 100644 --- a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts @@ -40,6 +40,16 @@ export class LogStreamEmbeddableFactoryDefinition }); } + public getDescription() { + return i18n.translate('xpack.infra.logStreamEmbeddable.description', { + defaultMessage: 'Add a table of live streaming logs.', + }); + } + + public getIconType() { + return 'logsApp'; + } + public async getExplicitInput() { return { title: i18n.translate('xpack.infra.logStreamEmbeddable.title', { diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable_factory.ts b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable_factory.ts index ac5ff2094e22b..4788d809f016f 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable_factory.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_charts/anomaly_charts_embeddable_factory.ts @@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n'; import type { StartServicesAccessor } from 'kibana/public'; +import { PLUGIN_ICON, PLUGIN_ID, ML_APP_NAME } from '../../../common/constants/app'; import type { EmbeddableFactoryDefinition, IContainer, @@ -27,6 +28,14 @@ export class AnomalyChartsEmbeddableFactory implements EmbeddableFactoryDefinition { public readonly type = ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE; + public readonly grouping = [ + { + id: PLUGIN_ID, + getDisplayName: () => ML_APP_NAME, + getIconType: () => PLUGIN_ICON, + }, + ]; + constructor( private getStartServices: StartServicesAccessor ) {} @@ -37,7 +46,13 @@ export class AnomalyChartsEmbeddableFactory public getDisplayName() { return i18n.translate('xpack.ml.components.mlAnomalyExplorerEmbeddable.displayName', { - defaultMessage: 'ML anomaly chart', + defaultMessage: 'Anomaly chart', + }); + } + + public getDescription() { + return i18n.translate('xpack.ml.components.mlAnomalyExplorerEmbeddable.description', { + defaultMessage: 'View anomaly detection results in a chart.', }); } diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts index fdb2ef8527923..bc45e075710c5 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts @@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n'; import type { StartServicesAccessor } from 'kibana/public'; +import { PLUGIN_ID, PLUGIN_ICON, ML_APP_NAME } from '../../../common/constants/app'; import type { EmbeddableFactoryDefinition, IContainer, @@ -26,6 +27,14 @@ export class AnomalySwimlaneEmbeddableFactory implements EmbeddableFactoryDefinition { public readonly type = ANOMALY_SWIMLANE_EMBEDDABLE_TYPE; + public readonly grouping = [ + { + id: PLUGIN_ID, + getDisplayName: () => ML_APP_NAME, + getIconType: () => PLUGIN_ICON, + }, + ]; + constructor( private getStartServices: StartServicesAccessor ) {} @@ -36,7 +45,13 @@ export class AnomalySwimlaneEmbeddableFactory public getDisplayName() { return i18n.translate('xpack.ml.components.jobAnomalyScoreEmbeddable.displayName', { - defaultMessage: 'ML anomaly swim lane', + defaultMessage: 'Anomaly swim lane', + }); + } + + public getDescription() { + return i18n.translate('xpack.ml.components.jobAnomalyScoreEmbeddable.description', { + defaultMessage: 'View anomaly detection results in a timeline.', }); } diff --git a/x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts b/x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts index deb91f6b9b1ef..51875c683346e 100644 --- a/x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts +++ b/x-pack/test/accessibility/apps/ml_embeddables_in_dashboard.ts @@ -96,8 +96,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('can open job selection flyout', async () => { await PageObjects.dashboard.clickCreateDashboardPrompt(); await ml.dashboardEmbeddables.assertDashboardIsEmpty(); - await dashboardAddPanel.clickOpenAddPanel(); - await dashboardAddPanel.ensureAddPanelIsShowing(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickEmbeddableFactoryGroupButton('ml'); await dashboardAddPanel.clickAddNewEmbeddableLink('ml_anomaly_charts'); await ml.dashboardJobSelectionTable.assertJobSelectionTableExists(); await a11y.testAppSnapshot(); diff --git a/x-pack/test/functional/apps/dashboard/dashboard_lens_by_value.ts b/x-pack/test/functional/apps/dashboard/dashboard_lens_by_value.ts index 56a8ab46a57da..87ecfe0dcada9 100644 --- a/x-pack/test/functional/apps/dashboard/dashboard_lens_by_value.ts +++ b/x-pack/test/functional/apps/dashboard/dashboard_lens_by_value.ts @@ -15,7 +15,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const testSubjects = getService('testSubjects'); const dashboardPanelActions = getService('dashboardPanelActions'); - const dashboardVisualizations = getService('dashboardVisualizations'); describe('dashboard lens by value', function () { before(async () => { @@ -27,7 +26,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); it('can add a lens panel by value', async () => { - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await PageObjects.lens.createAndAddLensFromDashboard({}); const newPanelCount = await PageObjects.dashboard.getPanelCount(); expect(newPanelCount).to.eql(1); diff --git a/x-pack/test/functional/apps/dashboard/dashboard_maps_by_value.ts b/x-pack/test/functional/apps/dashboard/dashboard_maps_by_value.ts index 15c76c3367a86..487dc90e1877e 100644 --- a/x-pack/test/functional/apps/dashboard/dashboard_maps_by_value.ts +++ b/x-pack/test/functional/apps/dashboard/dashboard_maps_by_value.ts @@ -19,10 +19,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const log = getService('log'); const esArchiver = getService('esArchiver'); - const dashboardVisualizations = getService('dashboardVisualizations'); const dashboardPanelActions = getService('dashboardPanelActions'); const testSubjects = getService('testSubjects'); const appsMenu = getService('appsMenu'); + const dashboardAddPanel = getService('dashboardAddPanel'); const LAYER_NAME = 'World Countries'; let mapCounter = 0; @@ -33,7 +33,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await PageObjects.visualize.clickMapsApp(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickVisType('maps'); await PageObjects.maps.clickSaveAndReturnButton(); } @@ -82,8 +83,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { describe('adding a map by value', () => { it('can add a map by value', async () => { await createNewDashboard(); - - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await createAndAddMapByValue(); const newPanelCount = await PageObjects.dashboard.getPanelCount(); expect(newPanelCount).to.eql(1); @@ -93,7 +92,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { describe('editing a map by value', () => { before(async () => { await createNewDashboard(); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await createAndAddMapByValue(); await editByValueMap(); }); @@ -112,7 +110,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { describe('editing a map and adding to map library', () => { beforeEach(async () => { await createNewDashboard(); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await createAndAddMapByValue(); }); diff --git a/x-pack/test/functional/apps/dashboard/feature_controls/time_to_visualize_security.ts b/x-pack/test/functional/apps/dashboard/feature_controls/time_to_visualize_security.ts index 3ebc53cc7cf27..730c00a8d5e4f 100644 --- a/x-pack/test/functional/apps/dashboard/feature_controls/time_to_visualize_security.ts +++ b/x-pack/test/functional/apps/dashboard/feature_controls/time_to_visualize_security.ts @@ -21,7 +21,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { 'lens', ]); - const dashboardVisualizations = getService('dashboardVisualizations'); + const dashboardAddPanel = getService('dashboardAddPanel'); const dashboardPanelActions = getService('dashboardPanelActions'); const dashboardExpect = getService('dashboardExpect'); const testSubjects = getService('testSubjects'); @@ -85,7 +85,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); it('can add a lens panel by value', async () => { - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await PageObjects.lens.createAndAddLensFromDashboard({}); const newPanelCount = await PageObjects.dashboard.getPanelCount(); expect(newPanelCount).to.eql(1); @@ -171,9 +170,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await PageObjects.dashboard.clickNewDashboard(); await PageObjects.dashboard.waitForRenderComplete(); - await testSubjects.click('dashboardAddNewPanelButton'); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickMarkdownWidget(); + await dashboardAddPanel.clickMarkdownQuickButton(); await PageObjects.visEditor.setMarkdownTxt(originalMarkdownText); await PageObjects.visEditor.clickGo(); diff --git a/x-pack/test/functional/apps/dashboard/sync_colors.ts b/x-pack/test/functional/apps/dashboard/sync_colors.ts index 7e54f966870c3..09575c355913e 100644 --- a/x-pack/test/functional/apps/dashboard/sync_colors.ts +++ b/x-pack/test/functional/apps/dashboard/sync_colors.ts @@ -49,7 +49,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await elasticChart.setNewChartUiDebugFlag(true); await PageObjects.dashboard.clickCreateDashboardPrompt(); await dashboardAddPanel.clickCreateNewLink(); - await dashboardAddPanel.clickVisType('lens'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.lens.goToTimeRange(); @@ -68,7 +67,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.save('vis1', false, true); await PageObjects.header.waitUntilLoadingHasFinished(); await dashboardAddPanel.clickCreateNewLink(); - await dashboardAddPanel.clickVisType('lens'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.lens.configureDimension({ diff --git a/x-pack/test/functional/apps/dashboard_mode/dashboard_empty_screen.js b/x-pack/test/functional/apps/dashboard_mode/dashboard_empty_screen.js index 57925ad50d155..37311de534195 100644 --- a/x-pack/test/functional/apps/dashboard_mode/dashboard_empty_screen.js +++ b/x-pack/test/functional/apps/dashboard_mode/dashboard_empty_screen.js @@ -10,7 +10,6 @@ import expect from '@kbn/expect'; export default function ({ getPageObjects, getService }) { const testSubjects = getService('testSubjects'); const esArchiver = getService('esArchiver'); - const dashboardVisualizations = getService('dashboardVisualizations'); const dashboardPanelActions = getService('dashboardPanelActions'); const PageObjects = getPageObjects(['common', 'dashboard', 'visualize', 'lens']); @@ -29,9 +28,6 @@ export default function ({ getPageObjects, getService }) { it('adds Lens visualization to empty dashboard', async () => { const title = 'Dashboard Test Lens'; - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await PageObjects.lens.createAndAddLensFromDashboard({ title, redirectToOrigin: true }); await PageObjects.dashboard.waitForRenderComplete(); await testSubjects.exists(`embeddablePanelHeading-${title}`); @@ -87,9 +83,6 @@ export default function ({ getPageObjects, getService }) { const title = 'non-dashboard Test Lens'; await PageObjects.dashboard.loadSavedDashboard('empty dashboard test'); await PageObjects.dashboard.switchToEditMode(); - await testSubjects.exists('dashboardAddNewPanelButton'); - await testSubjects.click('dashboardAddNewPanelButton'); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); await PageObjects.lens.createAndAddLensFromDashboard({ title }); await PageObjects.lens.notLinkedToOriginatingApp(); await PageObjects.common.navigateToApp('dashboard'); diff --git a/x-pack/test/functional/apps/lens/dashboard.ts b/x-pack/test/functional/apps/lens/dashboard.ts index a15176d76f953..1490abb320ca6 100644 --- a/x-pack/test/functional/apps/lens/dashboard.ts +++ b/x-pack/test/functional/apps/lens/dashboard.ts @@ -134,7 +134,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await filterBar.addFilter('geo.dest', 'is', 'LS'); await dashboardAddPanel.clickCreateNewLink(); - await dashboardAddPanel.clickVisType('lens'); await PageObjects.header.waitUntilLoadingHasFinished(); const hasGeoDestFilter = await filterBar.hasFilter('geo.dest', 'LS'); expect(hasGeoDestFilter).to.be(false); @@ -200,7 +199,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.dashboard.clickNewDashboard(); await dashboardAddPanel.clickCreateNewLink(); - await dashboardAddPanel.clickVisType('lens'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.lens.goToTimeRange(); diff --git a/x-pack/test/functional/apps/lens/lens_tagging.ts b/x-pack/test/functional/apps/lens/lens_tagging.ts index 7ce31709498fc..6fff2baa2d0cc 100644 --- a/x-pack/test/functional/apps/lens/lens_tagging.ts +++ b/x-pack/test/functional/apps/lens/lens_tagging.ts @@ -14,7 +14,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); const find = getService('find'); - const dashboardVisualizations = getService('dashboardVisualizations'); + const dashboardAddPanel = getService('dashboardAddPanel'); const dashboardPanelActions = getService('dashboardPanelActions'); const PageObjects = getPageObjects([ 'common', @@ -39,8 +39,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('adds a new tag to a Lens visualization', async () => { // create lens - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickLensWidget(); + await dashboardAddPanel.clickCreateNewLink(); await PageObjects.lens.goToTimeRange(); await PageObjects.lens.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', diff --git a/x-pack/test/functional/apps/maps/embeddable/embeddable_library.js b/x-pack/test/functional/apps/maps/embeddable/embeddable_library.js index 40e73f0d8a763..9bff4e56c6c5b 100644 --- a/x-pack/test/functional/apps/maps/embeddable/embeddable_library.js +++ b/x-pack/test/functional/apps/maps/embeddable/embeddable_library.js @@ -15,7 +15,6 @@ export default function ({ getPageObjects, getService }) { const security = getService('security'); const dashboardAddPanel = getService('dashboardAddPanel'); const dashboardPanelActions = getService('dashboardPanelActions'); - const dashboardVisualizations = getService('dashboardVisualizations'); describe('maps in embeddable library', () => { before(async () => { @@ -34,8 +33,7 @@ export default function ({ getPageObjects, getService }) { }); await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.clickNewDashboard(); - await dashboardAddPanel.clickCreateNewLink(); - await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); + await dashboardAddPanel.clickEditorMenuButton(); await PageObjects.visualize.clickMapsApp(); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.maps.waitForLayersToLoad(); diff --git a/x-pack/test/functional/apps/maps/embeddable/save_and_return.js b/x-pack/test/functional/apps/maps/embeddable/save_and_return.js index a3abb01b4cf9f..a7e649548306b 100644 --- a/x-pack/test/functional/apps/maps/embeddable/save_and_return.js +++ b/x-pack/test/functional/apps/maps/embeddable/save_and_return.js @@ -11,7 +11,6 @@ export default function ({ getPageObjects, getService }) { const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'maps', 'visualize']); const dashboardAddPanel = getService('dashboardAddPanel'); const dashboardPanelActions = getService('dashboardPanelActions'); - const dashboardVisualizations = getService('dashboardVisualizations'); const testSubjects = getService('testSubjects'); const security = getService('security'); @@ -37,9 +36,8 @@ export default function ({ getPageObjects, getService }) { beforeEach(async () => { await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.clickNewDashboard(); - await dashboardAddPanel.clickCreateNewLink(); - await await dashboardVisualizations.ensureNewVisualizationDialogIsShowing(); - await PageObjects.visualize.clickMapsApp(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickVisType('maps'); await PageObjects.header.waitUntilLoadingHasFinished(); await PageObjects.maps.waitForLayersToLoad(); }); diff --git a/x-pack/test/functional/apps/ml/embeddables/anomaly_charts_dashboard_embeddables.ts b/x-pack/test/functional/apps/ml/embeddables/anomaly_charts_dashboard_embeddables.ts index f7bfd7f7a4c62..0aee183c1a4a5 100644 --- a/x-pack/test/functional/apps/ml/embeddables/anomaly_charts_dashboard_embeddables.ts +++ b/x-pack/test/functional/apps/ml/embeddables/anomaly_charts_dashboard_embeddables.ts @@ -87,8 +87,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('can open job selection flyout', async () => { await PageObjects.dashboard.clickCreateDashboardPrompt(); await ml.dashboardEmbeddables.assertDashboardIsEmpty(); - await dashboardAddPanel.clickOpenAddPanel(); - await dashboardAddPanel.ensureAddPanelIsShowing(); + await dashboardAddPanel.clickEditorMenuButton(); + await dashboardAddPanel.clickEmbeddableFactoryGroupButton('ml'); await dashboardAddPanel.clickAddNewEmbeddableLink('ml_anomaly_charts'); await ml.dashboardJobSelectionTable.assertJobSelectionTableExists(); }); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 65020be390f9d..100ed8e079d37 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -18,6 +18,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont const find = getService('find'); const comboBox = getService('comboBox'); const browser = getService('browser'); + const dashboardAddPanel = getService('dashboardAddPanel'); const PageObjects = getPageObjects([ 'common', @@ -753,7 +754,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont if (inViewMode) { await PageObjects.dashboard.switchToEditMode(); } - await PageObjects.visualize.clickLensWidget(); + await dashboardAddPanel.clickCreateNewLink(); await this.goToTimeRange(); await this.configureDimension({ dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', From 1a3e033c90eb9217168073abddcca2e11220009a Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Sun, 18 Apr 2021 12:50:02 +0300 Subject: [PATCH 20/25] [Partial Results] Move other bucket into Search Source (#96384) * Move inspector adapter integration into search source * docs and ts * Move other bucket to search source * test ts + delete unused tabilfy function * hierarchical param in aggconfig. ts improvements more inspector tests * fix jest * separate inspect more tests * jest * inspector * Error handling and more tests * put the fun in functional tests * code review * Add functional test for other bucket in search example app * test * test * ts * test * test * ts Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...ins-data-public.aggconfigs.hierarchical.md | 11 + ...a-plugin-plugins-data-public.aggconfigs.md | 3 +- ...in-plugins-data-public.aggconfigs.todsl.md | 9 +- ...s-data-public.isearchoptions.inspector.md} | 8 +- ...ugin-plugins-data-public.isearchoptions.md | 2 +- ...-plugins-data-public.searchsource.fetch.md | 4 +- ...plugins-data-public.searchsource.fetch_.md | 4 +- ...ins-data-public.searchsourcefields.aggs.md | 2 +- ...-plugins-data-public.searchsourcefields.md | 2 +- ...s-data-server.isearchoptions.inspector.md} | 8 +- ...ugin-plugins-data-server.isearchoptions.md | 2 +- examples/search_examples/public/index.scss | 6 + .../search_examples/public/search/app.tsx | 274 ++++++---- .../common/search/aggs/agg_configs.test.ts | 8 +- .../data/common/search/aggs/agg_configs.ts | 8 +- .../data/common/search/aggs/agg_type.ts | 31 +- .../_terms_other_bucket_helper.test.ts | 6 +- .../buckets/_terms_other_bucket_helper.ts | 13 +- .../data/common/search/aggs/buckets/terms.ts | 26 +- .../esaggs/request_handler.test.ts | 43 +- .../expressions/esaggs/request_handler.ts | 62 +-- .../search_source/inspect/inspector_stats.ts | 2 +- .../search_source/search_source.test.ts | 482 +++++++++++++++--- .../search/search_source/search_source.ts | 179 ++++++- .../data/common/search/search_source/types.ts | 3 +- .../data/common/search/tabify/index.ts | 23 +- src/plugins/data/common/search/types.ts | 15 +- src/plugins/data/public/public.api.md | 17 +- .../public/search/expressions/esaggs.test.ts | 7 +- .../data/public/search/expressions/esaggs.ts | 7 +- .../server/search/expressions/esaggs.test.ts | 6 +- .../data/server/search/expressions/esaggs.ts | 8 +- src/plugins/data/server/server.api.md | 8 +- .../public/application/angular/discover.js | 22 +- .../embeddable/search_embeddable.ts | 22 +- .../classes/sources/es_source/es_source.ts | 13 +- x-pack/test/examples/search_examples/index.ts | 3 +- .../search_examples/search_example.ts | 38 ++ 38 files changed, 964 insertions(+), 423 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md rename docs/development/plugins/data/public/{kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md => kibana-plugin-plugins-data-public.isearchoptions.inspector.md} (52%) rename docs/development/plugins/data/server/{kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md => kibana-plugin-plugins-data-server.isearchoptions.inspector.md} (52%) create mode 100644 x-pack/test/examples/search_examples/search_example.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md new file mode 100644 index 0000000000000..66d540c48c3bc --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) + +## AggConfigs.hierarchical property + +Signature: + +```typescript +hierarchical?: boolean; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index 22f8994747aa2..02e9a63d95ba3 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -22,6 +22,7 @@ export declare class AggConfigs | --- | --- | --- | --- | | [aggs](./kibana-plugin-plugins-data-public.aggconfigs.aggs.md) | | IAggConfig[] | | | [createAggConfig](./kibana-plugin-plugins-data-public.aggconfigs.createaggconfig.md) | | <T extends AggConfig = AggConfig>(params: CreateAggConfigParams, { addToAggConfigs }?: {
addToAggConfigs?: boolean | undefined;
}) => T | | +| [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) | | boolean | | | [indexPattern](./kibana-plugin-plugins-data-public.aggconfigs.indexpattern.md) | | IndexPattern | | | [timeFields](./kibana-plugin-plugins-data-public.aggconfigs.timefields.md) | | string[] | | | [timeRange](./kibana-plugin-plugins-data-public.aggconfigs.timerange.md) | | TimeRange | | @@ -46,5 +47,5 @@ export declare class AggConfigs | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | -| [toDsl(hierarchical)](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | +| [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md index 055c4113ca3e4..1327e976db0ce 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.todsl.md @@ -7,15 +7,8 @@ Signature: ```typescript -toDsl(hierarchical?: boolean): Record; +toDsl(): Record; ``` - -## Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| hierarchical | boolean | | - Returns: `Record` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md similarity index 52% rename from docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md rename to docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md index b4431b9467b71..9961292aaf217 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.inspector.md @@ -1,11 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md) -## ISearchOptions.requestResponder property +## ISearchOptions.inspector property + +Inspector integration options Signature: ```typescript -requestResponder?: RequestResponder; +inspector?: IInspectorInfo; ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md index cc0cb538be611..21fb7e3dfc7e8 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md @@ -16,10 +16,10 @@ export interface ISearchOptions | --- | --- | --- | | [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | AbortSignal | An AbortSignal that allows the caller of search to abort a search request. | | [indexPattern](./kibana-plugin-plugins-data-public.isearchoptions.indexpattern.md) | IndexPattern | Index pattern reference is used for better error messages | +| [inspector](./kibana-plugin-plugins-data-public.isearchoptions.inspector.md) | IInspectorInfo | Inspector integration options | | [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | -| [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md index 623d6366d4d13..e6ba1a51a867d 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch.md @@ -14,7 +14,7 @@ Fetch this source and reject the returned Promise on error Signature: ```typescript -fetch(options?: ISearchOptions): Promise>; +fetch(options?: ISearchOptions): Promise>; ``` ## Parameters @@ -25,5 +25,5 @@ fetch(options?: ISearchOptions): PromiseReturns: -`Promise>` +`Promise>` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md index d5641107a88aa..4369cf7c087da 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.fetch_.md @@ -9,7 +9,7 @@ Fetch this source from Elasticsearch, returning an observable over the response( Signature: ```typescript -fetch$(options?: ISearchOptions): import("rxjs").Observable>; +fetch$(options?: ISearchOptions): Observable>; ``` ## Parameters @@ -20,5 +20,5 @@ fetch$(options?: ISearchOptions): import("rxjs").ObservableReturns: -`import("rxjs").Observable>` +`Observable>` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md index f6bab8e424857..12011f8242996 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.aggs.md @@ -9,5 +9,5 @@ Signature: ```typescript -aggs?: any; +aggs?: object | IAggConfigs | (() => object); ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md index d0f53936eb56a..981d956a9e89b 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsourcefields.md @@ -16,7 +16,7 @@ export interface SearchSourceFields | Property | Type | Description | | --- | --- | --- | -| [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | any | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) | +| [aggs](./kibana-plugin-plugins-data-public.searchsourcefields.aggs.md) | object | IAggConfigs | (() => object) | [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) | | [fields](./kibana-plugin-plugins-data-public.searchsourcefields.fields.md) | SearchFieldValue[] | Retrieve fields via the search Fields API | | [fieldsFromSource](./kibana-plugin-plugins-data-public.searchsourcefields.fieldsfromsource.md) | NameList | Retreive fields directly from \_source (legacy behavior) | | [filter](./kibana-plugin-plugins-data-public.searchsourcefields.filter.md) | Filter[] | Filter | (() => Filter[] | Filter | undefined) | [Filter](./kibana-plugin-plugins-data-public.filter.md) | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md similarity index 52% rename from docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md rename to docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md index 7440f5a9d26cf..ab755334643aa 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.inspector.md @@ -1,11 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md) -## ISearchOptions.requestResponder property +## ISearchOptions.inspector property + +Inspector integration options Signature: ```typescript -requestResponder?: RequestResponder; +inspector?: IInspectorInfo; ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md index 413a59be3d427..cdb5664f96cdd 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md @@ -16,10 +16,10 @@ export interface ISearchOptions | --- | --- | --- | | [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | AbortSignal | An AbortSignal that allows the caller of search to abort a search request. | | [indexPattern](./kibana-plugin-plugins-data-server.isearchoptions.indexpattern.md) | IndexPattern | Index pattern reference is used for better error messages | +| [inspector](./kibana-plugin-plugins-data-server.isearchoptions.inspector.md) | IInspectorInfo | Inspector integration options | | [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | -| [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/examples/search_examples/public/index.scss b/examples/search_examples/public/index.scss index e69de29bb2d1d..b623fecf78640 100644 --- a/examples/search_examples/public/index.scss +++ b/examples/search_examples/public/index.scss @@ -0,0 +1,6 @@ +@import '@elastic/eui/src/global_styling/variables/header'; + +.searchExampleStepDsc { + padding-left: $euiSizeXL; + font-style: italic; +} diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index 8f31d242faf5e..b2a4991d0717b 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -20,13 +20,13 @@ import { EuiTitle, EuiText, EuiFlexGrid, - EuiFlexGroup, EuiFlexItem, EuiCheckbox, EuiSpacer, EuiCode, EuiComboBox, EuiFormLabel, + EuiTabbedContent, } from '@elastic/eui'; import { CoreStart } from '../../../../src/core/public'; @@ -60,6 +60,11 @@ function getNumeric(fields?: IndexPatternField[]) { return fields?.filter((f) => f.type === 'number' && f.aggregatable); } +function getAggregatableStrings(fields?: IndexPatternField[]) { + if (!fields) return []; + return fields?.filter((f) => f.type === 'string' && f.aggregatable); +} + function formatFieldToComboBox(field?: IndexPatternField | null) { if (!field) return []; return formatFieldsToComboBox([field]); @@ -90,6 +95,9 @@ export const SearchExamplesApp = ({ const [selectedNumericField, setSelectedNumericField] = useState< IndexPatternField | null | undefined >(); + const [selectedBucketField, setSelectedBucketField] = useState< + IndexPatternField | null | undefined + >(); const [request, setRequest] = useState>({}); const [response, setResponse] = useState>({}); @@ -108,6 +116,7 @@ export const SearchExamplesApp = ({ setFields(indexPattern?.fields); }, [indexPattern]); useEffect(() => { + setSelectedBucketField(fields?.length ? getAggregatableStrings(fields)[0] : null); setSelectedNumericField(fields?.length ? getNumeric(fields)[0] : null); }, [fields]); @@ -203,28 +212,40 @@ export const SearchExamplesApp = ({ .setField('index', indexPattern) .setField('filter', filters) .setField('query', query) - .setField('fields', selectedFields.length ? selectedFields.map((f) => f.name) : ['*']) + .setField('fields', selectedFields.length ? selectedFields.map((f) => f.name) : ['']) + .setField('size', selectedFields.length ? 100 : 0) .setField('trackTotalHits', 100); - if (selectedNumericField) { - searchSource.setField('aggs', () => { - return data.search.aggs - .createAggConfigs(indexPattern, [ - { type: 'avg', params: { field: selectedNumericField.name } }, - ]) - .toDsl(); + const aggDef = []; + if (selectedBucketField) { + aggDef.push({ + type: 'terms', + schema: 'split', + params: { field: selectedBucketField.name, size: 2, otherBucket: true }, }); } + if (selectedNumericField) { + aggDef.push({ type: 'avg', params: { field: selectedNumericField.name } }); + } + if (aggDef.length > 0) { + const ac = data.search.aggs.createAggConfigs(indexPattern, aggDef); + searchSource.setField('aggs', ac); + } setRequest(searchSource.getSearchRequestBody()); const res = await searchSource.fetch$().toPromise(); setResponse(res); const message = Searched {res.hits.total} documents.; - notifications.toasts.addSuccess({ - title: 'Query result', - text: mountReactNode(message), - }); + notifications.toasts.addSuccess( + { + title: 'Query result', + text: mountReactNode(message), + }, + { + toastLifeTimeMs: 300000, + } + ); } catch (e) { setResponse(e.body); notifications.toasts.addWarning(`An error has occurred: ${e.message}`); @@ -263,6 +284,55 @@ export const SearchExamplesApp = ({ doSearchSourceSearch(); }; + const reqTabs = [ + { + id: 'request', + name: Request, + content: ( + <> + + Search body sent to ES + + {JSON.stringify(request, null, 2)} + + + ), + }, + { + id: 'response', + name: Response, + content: ( + <> + + + + + + {JSON.stringify(response, null, 2)} + + + ), + }, + ]; + return ( @@ -284,59 +354,75 @@ export const SearchExamplesApp = ({ useDefaultBehaviors={true} indexPatterns={indexPattern ? [indexPattern] : undefined} /> - + + + Index Pattern + { + const newIndexPattern = await data.indexPatterns.get(newIndexPatternId); + setIndexPattern(newIndexPattern); + }} + isClearable={false} + /> + + + Field (bucket) + { + if (option.length) { + const fld = indexPattern?.getFieldByName(option[0].label); + setSelectedBucketField(fld || null); + } else { + setSelectedBucketField(null); + } + }} + sortMatchesBy="startsWith" + data-test-subj="searchBucketField" + /> + + + Numeric Field (metric) + { + if (option.length) { + const fld = indexPattern?.getFieldByName(option[0].label); + setSelectedNumericField(fld || null); + } else { + setSelectedNumericField(null); + } + }} + sortMatchesBy="startsWith" + data-test-subj="searchMetricField" + /> + + + Fields to queryString + { + const flds = option + .map((opt) => indexPattern?.getFieldByName(opt?.label)) + .filter((f) => f); + setSelectedFields(flds.length ? (flds as IndexPatternField[]) : []); + }} + sortMatchesBy="startsWith" + /> + + + - - - - Index Pattern - { - const newIndexPattern = await data.indexPatterns.get(newIndexPatternId); - setIndexPattern(newIndexPattern); - }} - isClearable={false} - /> - - - Numeric Field to Aggregate - { - const fld = indexPattern?.getFieldByName(option[0].label); - setSelectedNumericField(fld || null); - }} - sortMatchesBy="startsWith" - /> - - - - - Fields to query (leave blank to include all fields) - { - const flds = option - .map((opt) => indexPattern?.getFieldByName(opt?.label)) - .filter((f) => f); - setSelectedFields(flds.length ? (flds as IndexPatternField[]) : []); - }} - sortMatchesBy="startsWith" - /> - - -

@@ -352,15 +438,32 @@ export const SearchExamplesApp = ({ - + + + + + + + @@ -446,41 +549,8 @@ export const SearchExamplesApp = ({ - - -

Request

- - Search body sent to ES - - {JSON.stringify(request, null, 2)} - - - - -

Response

- - - - - - {JSON.stringify(response, null, 2)} - + + diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index 3ce528e6ed893..28102544ae055 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -342,8 +342,8 @@ describe('AggConfigs', () => { { enabled: true, type: 'max', schema: 'metric', params: { field: 'bytes' } }, ]; - const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); - const topLevelDsl = ac.toDsl(true); + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry, hierarchical: true }); + const topLevelDsl = ac.toDsl(); const buckets = ac.bySchemaName('buckets'); const metrics = ac.bySchemaName('metrics'); @@ -412,8 +412,8 @@ describe('AggConfigs', () => { }, ]; - const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); - const topLevelDsl = ac.toDsl(true)['2']; + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry, hierarchical: true }); + const topLevelDsl = ac.toDsl()['2']; expect(Object.keys(topLevelDsl.aggs)).toContain('1'); expect(Object.keys(topLevelDsl.aggs)).toContain('1-bucket'); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 4d5d49754387d..2932ef7325aed 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -43,6 +43,7 @@ function parseParentAggs(dslLvlCursor: any, dsl: any) { export interface AggConfigsOptions { typesRegistry: AggTypesRegistryStart; + hierarchical?: boolean; } export type CreateAggConfigParams = Assign; @@ -65,6 +66,8 @@ export class AggConfigs { public indexPattern: IndexPattern; public timeRange?: TimeRange; public timeFields?: string[]; + public hierarchical?: boolean = false; + private readonly typesRegistry: AggTypesRegistryStart; aggs: IAggConfig[]; @@ -80,6 +83,7 @@ export class AggConfigs { this.aggs = []; this.indexPattern = indexPattern; + this.hierarchical = opts.hierarchical; configStates.forEach((params: any) => this.createAggConfig(params)); } @@ -174,12 +178,12 @@ export class AggConfigs { return true; } - toDsl(hierarchical: boolean = false): Record { + toDsl(): Record { const dslTopLvl = {}; let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; - if (hierarchical) { + if (this.hierarchical) { // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs .filter(function (agg) { diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index 33fdc45a605b7..f0f3912bf64fe 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -13,12 +13,23 @@ import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; import type { RequestAdapter } from 'src/plugins/inspector/common'; +import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; import { IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; +type PostFlightRequestFn = ( + resp: estypes.SearchResponse, + aggConfigs: IAggConfigs, + aggConfig: TAggConfig, + searchSource: ISearchSource, + inspectorRequestAdapter?: RequestAdapter, + abortSignal?: AbortSignal, + searchSessionId?: string +) => Promise>; + export interface AggTypeConfig< TAggConfig extends AggConfig = AggConfig, TParam extends AggParamType = AggParamType @@ -40,15 +51,7 @@ export interface AggTypeConfig< customLabels?: boolean; json?: boolean; decorateAggConfig?: () => any; - postFlightRequest?: ( - resp: any, - aggConfigs: IAggConfigs, - aggConfig: TAggConfig, - searchSource: ISearchSource, - inspectorRequestAdapter?: RequestAdapter, - abortSignal?: AbortSignal, - searchSessionId?: string - ) => Promise; + postFlightRequest?: PostFlightRequestFn; getSerializedFormat?: (agg: TAggConfig) => SerializedFieldFormat; getValue?: (agg: TAggConfig, bucket: any) => any; getKey?: (bucket: any, key: any, agg: TAggConfig) => any; @@ -188,15 +191,7 @@ export class AggType< * @param searchSessionId - searchSessionId to be used for grouping requests into a single search session * @return {Promise} */ - postFlightRequest: ( - resp: any, - aggConfigs: IAggConfigs, - aggConfig: TAggConfig, - searchSource: ISearchSource, - inspectorRequestAdapter?: RequestAdapter, - abortSignal?: AbortSignal, - searchSessionId?: string - ) => Promise; + postFlightRequest: PostFlightRequestFn; /** * Get the serialized format for the values produced by this agg type, * overridden by several metrics that always output a simple number. diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 56e720d237c45..2aa0d346afe34 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -433,7 +433,7 @@ describe('Terms Agg Other bucket helper', () => { aggConfigs.aggs[0] as IBucketAggConfig, otherAggConfig() ); - expect(mergedResponse.aggregations['1'].buckets[3].key).toEqual('__other__'); + expect((mergedResponse!.aggregations!['1'] as any).buckets[3].key).toEqual('__other__'); } }); @@ -455,7 +455,7 @@ describe('Terms Agg Other bucket helper', () => { otherAggConfig() ); - expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).toEqual( + expect((mergedResponse!.aggregations!['1'] as any).buckets[1]['2'].buckets[3].key).toEqual( '__other__' ); } @@ -471,7 +471,7 @@ describe('Terms Agg Other bucket helper', () => { aggConfigs.aggs[0] as IBucketAggConfig ); expect( - updatedResponse.aggregations['1'].buckets.find( + (updatedResponse!.aggregations!['1'] as any).buckets.find( (bucket: Record) => bucket.key === '__missing__' ) ).toBeDefined(); diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 742615bc49d8f..6230ae897b170 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -7,6 +7,7 @@ */ import { isNumber, keys, values, find, each, cloneDeep, flatten } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; import { buildExistsFilter, buildPhrasesFilter, buildQueryFromFilters } from '../../../../common'; import { AggGroupNames } from '../agg_groups'; import { IAggConfigs } from '../agg_configs'; @@ -42,7 +43,7 @@ const getNestedAggDSL = (aggNestedDsl: Record, startFromAggId: stri */ const getAggResultBuckets = ( aggConfigs: IAggConfigs, - response: any, + response: estypes.SearchResponse['aggregations'], aggWithOtherBucket: IBucketAggConfig, key: string ) => { @@ -72,8 +73,8 @@ const getAggResultBuckets = ( } } } - if (responseAgg[aggWithOtherBucket.id]) { - return responseAgg[aggWithOtherBucket.id].buckets; + if (responseAgg?.[aggWithOtherBucket.id]) { + return (responseAgg[aggWithOtherBucket.id] as any).buckets; } return []; }; @@ -235,11 +236,11 @@ export const buildOtherBucketAgg = ( export const mergeOtherBucketAggResponse = ( aggsConfig: IAggConfigs, - response: any, + response: estypes.SearchResponse, otherResponse: any, otherAgg: IBucketAggConfig, requestAgg: Record -) => { +): estypes.SearchResponse => { const updatedResponse = cloneDeep(response); each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { if (!bucket.doc_count || key === undefined) return; @@ -276,7 +277,7 @@ export const mergeOtherBucketAggResponse = ( }; export const updateMissingBucket = ( - response: any, + response: estypes.SearchResponse, aggConfigs: IAggConfigs, agg: IBucketAggConfig ) => { diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 77c9c6e391c0a..03cf14a577a50 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -101,25 +101,21 @@ export const getTermsBucketAgg = () => nestedSearchSource.setField('aggs', filterAgg); - const requestResponder = inspectorRequestAdapter?.start( - i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { - defaultMessage: 'Other bucket', - }), - { - description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { - defaultMessage: - 'This request counts the number of documents that fall ' + - 'outside the criterion of the data buckets.', - }), - searchSessionId, - } - ); - const response = await nestedSearchSource .fetch$({ abortSignal, sessionId: searchSessionId, - requestResponder, + inspector: { + adapter: inspectorRequestAdapter, + title: i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { + defaultMessage: 'Other bucket', + }), + description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { + defaultMessage: + 'This request counts the number of documents that fall ' + + 'outside the criterion of the data buckets.', + }), + }, }) .toPromise(); diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts index c2566535916a8..b30e5740fa3fb 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts @@ -9,7 +9,7 @@ import type { MockedKeys } from '@kbn/utility-types/jest'; import type { Filter } from '../../../es_query'; import type { IndexPattern } from '../../../index_patterns'; -import type { IAggConfig, IAggConfigs } from '../../aggs'; +import type { IAggConfigs } from '../../aggs'; import type { ISearchSource } from '../../search_source'; import { searchSourceCommonMock } from '../../search_source/mocks'; @@ -38,7 +38,6 @@ describe('esaggs expression function - public', () => { filters: undefined, indexPattern: ({ id: 'logstash-*' } as unknown) as jest.Mocked, inspectorAdapters: {}, - metricsAtAllLevels: false, partialRows: false, query: undefined, searchSessionId: 'abc123', @@ -76,21 +75,7 @@ describe('esaggs expression function - public', () => { test('setField(aggs)', async () => { expect(searchSource.setField).toHaveBeenCalledTimes(5); - expect(typeof (searchSource.setField as jest.Mock).mock.calls[2][1]).toBe('function'); - expect((searchSource.setField as jest.Mock).mock.calls[2][1]()).toEqual( - mockParams.aggs.toDsl() - ); - expect(mockParams.aggs.toDsl).toHaveBeenCalledWith(mockParams.metricsAtAllLevels); - - // make sure param is passed through - jest.clearAllMocks(); - await handleRequest({ - ...mockParams, - metricsAtAllLevels: true, - }); - searchSource = await mockParams.searchSourceService.create(); - (searchSource.setField as jest.Mock).mock.calls[2][1](); - expect(mockParams.aggs.toDsl).toHaveBeenCalledWith(true); + expect((searchSource.setField as jest.Mock).mock.calls[2][1]).toEqual(mockParams.aggs); }); test('setField(filter)', async () => { @@ -133,36 +118,24 @@ describe('esaggs expression function - public', () => { test('calls searchSource.fetch', async () => { await handleRequest(mockParams); const searchSource = await mockParams.searchSourceService.create(); + expect(searchSource.fetch$).toHaveBeenCalledWith({ abortSignal: mockParams.abortSignal, sessionId: mockParams.searchSessionId, + inspector: { + title: 'Data', + description: 'This request queries Elasticsearch to fetch the data for the visualization.', + adapter: undefined, + }, }); }); - test('calls agg.postFlightRequest if it exiests and agg is enabled', async () => { - mockParams.aggs.aggs[0].enabled = true; - await handleRequest(mockParams); - expect(mockParams.aggs.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(1); - - // ensure it works if the function doesn't exist - jest.clearAllMocks(); - mockParams.aggs.aggs[0] = ({ type: { name: 'count' } } as unknown) as IAggConfig; - expect(async () => await handleRequest(mockParams)).not.toThrowError(); - }); - - test('should skip agg.postFlightRequest call if the agg is disabled', async () => { - mockParams.aggs.aggs[0].enabled = false; - await handleRequest(mockParams); - expect(mockParams.aggs.aggs[0].type.postFlightRequest).toHaveBeenCalledTimes(0); - }); - test('tabifies response data', async () => { await handleRequest(mockParams); expect(tabifyAggResponse).toHaveBeenCalledWith( mockParams.aggs, {}, { - metricsAtAllLevels: mockParams.metricsAtAllLevels, partialRows: mockParams.partialRows, timeRange: mockParams.timeRange, } diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 5620698a47538..173b2067cad6b 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -40,28 +40,12 @@ export interface RequestHandlerParams { getNow?: () => Date; } -function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) { - return inspectorAdapters.requests?.start( - i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { - defaultMessage: 'Data', - }), - { - description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { - defaultMessage: - 'This request queries Elasticsearch to fetch the data for the visualization.', - }), - searchSessionId, - } - ); -} - export const handleRequest = async ({ abortSignal, aggs, filters, indexPattern, inspectorAdapters, - metricsAtAllLevels, partialRows, query, searchSessionId, @@ -100,9 +84,7 @@ export const handleRequest = async ({ }, }); - requestSearchSource.setField('aggs', function () { - return aggs.toDsl(metricsAtAllLevels); - }); + requestSearchSource.setField('aggs', aggs); requestSearchSource.onRequestStart((paramSearchSource, options) => { return aggs.onSearchRequestStart(paramSearchSource, options); @@ -128,35 +110,27 @@ export const handleRequest = async ({ requestSearchSource.setField('query', query); inspectorAdapters.requests?.reset(); - const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId); - const response$ = await requestSearchSource.fetch$({ - abortSignal, - sessionId: searchSessionId, - requestResponder, - }); - - // Note that rawResponse is not deeply cloned here, so downstream applications using courier - // must take care not to mutate it, or it could have unintended side effects, e.g. displaying - // response data incorrectly in the inspector. - let response = await response$.toPromise(); - for (const agg of aggs.aggs) { - if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { - response = await agg.type.postFlightRequest( - response, - aggs, - agg, - requestSearchSource, - inspectorAdapters.requests, - abortSignal, - searchSessionId - ); - } - } + const response = await requestSearchSource + .fetch$({ + abortSignal, + sessionId: searchSessionId, + inspector: { + adapter: inspectorAdapters.requests, + title: i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { + defaultMessage: 'Data', + }), + description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the visualization.', + }), + }, + }) + .toPromise(); const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null; const tabifyParams = { - metricsAtAllLevels, + metricsAtAllLevels: aggs.hierarchical, partialRows, timeRange: parsedTimeRange ? { from: parsedTimeRange.min, to: parsedTimeRange.max, timeFields: allTimeFields } diff --git a/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts index 24507a7e13058..e5a3acc23eee8 100644 --- a/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts +++ b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts @@ -50,7 +50,7 @@ export function getRequestInspectorStats(searchSource: ISearchSource) { /** @public */ export function getResponseInspectorStats( - resp: estypes.SearchResponse, + resp?: estypes.SearchResponse, searchSource?: ISearchSource ) { const lastRequest = diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 3726e5d0c33e8..7f8a4fceff05d 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -11,6 +11,10 @@ import { IndexPattern } from '../../index_patterns'; import { GetConfigFn } from '../../types'; import { fetchSoon } from './legacy'; import { SearchSource, SearchSourceDependencies, SortDirection } from './'; +import { AggConfigs, AggTypesRegistryStart } from '../../'; +import { mockAggTypesRegistry } from '../aggs/test_helpers'; +import { RequestResponder } from 'src/plugins/inspector/common'; +import { switchMap } from 'rxjs/operators'; jest.mock('./legacy', () => ({ fetchSoon: jest.fn().mockResolvedValue({}), @@ -39,6 +43,21 @@ const indexPattern2 = ({ getSourceFiltering: () => mockSource2, } as unknown) as IndexPattern; +const fields3 = [{ name: 'foo-bar' }, { name: 'field1' }, { name: 'field2' }]; +const indexPattern3 = ({ + title: 'foo', + fields: { + getByName: (name: string) => { + return fields3.find((field) => field.name === name); + }, + filter: () => { + return fields3; + }, + }, + getComputedFields, + getSourceFiltering: () => mockSource, +} as unknown) as IndexPattern; + const runtimeFieldDef = { type: 'keyword', script: { @@ -61,8 +80,8 @@ describe('SearchSource', () => { .fn() .mockReturnValue( of( - { rawResponse: { isPartial: true, isRunning: true } }, - { rawResponse: { isPartial: false, isRunning: false } } + { rawResponse: { test: 1 }, isPartial: true, isRunning: true }, + { rawResponse: { test: 2 }, isPartial: false, isRunning: false } ) ); @@ -81,17 +100,19 @@ describe('SearchSource', () => { describe('#getField()', () => { test('gets the value for the property', () => { - searchSource.setField('aggs', 5); - expect(searchSource.getField('aggs')).toBe(5); + searchSource.setField('aggs', { i: 5 }); + expect(searchSource.getField('aggs')).toStrictEqual({ i: 5 }); }); }); describe('#getFields()', () => { test('gets the value for the property', () => { - searchSource.setField('aggs', 5); + searchSource.setField('aggs', { i: 5 }); expect(searchSource.getFields()).toMatchInlineSnapshot(` Object { - "aggs": 5, + "aggs": Object { + "i": 5, + }, } `); }); @@ -100,7 +121,7 @@ describe('SearchSource', () => { describe('#removeField()', () => { test('remove property', () => { searchSource = new SearchSource({}, searchSourceDependencies); - searchSource.setField('aggs', 5); + searchSource.setField('aggs', { i: 5 }); searchSource.removeField('aggs'); expect(searchSource.getField('aggs')).toBeFalsy(); }); @@ -108,8 +129,20 @@ describe('SearchSource', () => { describe('#setField() / #flatten', () => { test('sets the value for the property', () => { - searchSource.setField('aggs', 5); - expect(searchSource.getField('aggs')).toBe(5); + searchSource.setField('aggs', { i: 5 }); + expect(searchSource.getField('aggs')).toStrictEqual({ i: 5 }); + }); + + test('sets the value for the property with AggConfigs', () => { + const typesRegistry = mockAggTypesRegistry(); + + const ac = new AggConfigs(indexPattern3, [{ type: 'avg', params: { field: 'field1' } }], { + typesRegistry, + }); + + searchSource.setField('aggs', ac); + const request = searchSource.getSearchRequestBody(); + expect(request.aggs).toStrictEqual({ '1': { avg: { field: 'field1' } } }); }); describe('computed fields handling', () => { @@ -631,7 +664,7 @@ describe('SearchSource', () => { const fn = jest.fn(); searchSource.onRequestStart(fn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); }); @@ -644,7 +677,7 @@ describe('SearchSource', () => { const parentFn = jest.fn(); parent.onRequestStart(parentFn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); expect(parentFn).not.toBeCalled(); @@ -664,69 +697,13 @@ describe('SearchSource', () => { const parentFn = jest.fn(); parent.onRequestStart(parentFn); const options = {}; - await searchSource.fetch(options); + await searchSource.fetch$(options).toPromise(); expect(fn).toBeCalledWith(searchSource, options); expect(parentFn).toBeCalledWith(searchSource, options); }); }); - describe('#legacy fetch()', () => { - beforeEach(() => { - searchSourceDependencies = { - ...searchSourceDependencies, - getConfig: jest.fn(() => { - return true; // batchSearches = true - }) as GetConfigFn, - }; - }); - - test('should call msearch', async () => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - await searchSource.fetch(options); - expect(fetchSoon).toBeCalledTimes(1); - }); - }); - - describe('#search service fetch()', () => { - test('should call msearch', async () => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - - await searchSource.fetch(options); - expect(mockSearchMethod).toBeCalledTimes(1); - }); - - test('should return partial results', (done) => { - searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); - const options = {}; - - const next = jest.fn(); - const complete = () => { - expect(next).toBeCalledTimes(2); - expect(next.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Object { - "isPartial": true, - "isRunning": true, - }, - ] - `); - expect(next.mock.calls[1]).toMatchInlineSnapshot(` - Array [ - Object { - "isPartial": false, - "isRunning": false, - }, - ] - `); - done(); - }; - searchSource.fetch$(options).subscribe({ next, complete }); - }); - }); - describe('#serialize', () => { test('should reference index patterns', () => { const indexPattern123 = { id: '123' } as IndexPattern; @@ -884,4 +861,373 @@ describe('SearchSource', () => { ); }); }); + + describe('fetch$', () => { + describe('#legacy fetch()', () => { + beforeEach(() => { + searchSourceDependencies = { + ...searchSourceDependencies, + getConfig: jest.fn(() => { + return true; // batchSearches = true + }) as GetConfigFn, + }; + }); + + test('should call msearch', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + await searchSource.fetch$(options).toPromise(); + expect(fetchSoon).toBeCalledTimes(1); + }); + }); + + describe('responses', () => { + test('should return partial results', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + + const next = jest.fn(); + const complete = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, complete }); + await res$.toPromise(); + + expect(next).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(1); + expect(next.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 1, + }, + ] + `); + expect(next.mock.calls[1]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 2, + }, + ] + `); + }); + + test('shareReplays result', async () => { + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + + const next = jest.fn(); + const complete = jest.fn(); + const next2 = jest.fn(); + const complete2 = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, complete }); + res$.subscribe({ next: next2, complete: complete2 }); + await res$.toPromise(); + + expect(next).toBeCalledTimes(2); + expect(next2).toBeCalledTimes(2); + expect(complete).toBeCalledTimes(1); + expect(complete2).toBeCalledTimes(1); + expect(searchSourceDependencies.search).toHaveBeenCalledTimes(1); + }); + + test('should emit error on empty response', async () => { + searchSourceDependencies.search = mockSearchMethod = jest + .fn() + .mockReturnValue( + of({ rawResponse: { test: 1 }, isPartial: true, isRunning: true }, undefined) + ); + + searchSource = new SearchSource({ index: indexPattern }, searchSourceDependencies); + const options = {}; + + const next = jest.fn(); + const error = jest.fn(); + const complete = jest.fn(); + const res$ = searchSource.fetch$(options); + res$.subscribe({ next, error, complete }); + await res$.toPromise().catch((e) => {}); + + expect(next).toBeCalledTimes(1); + expect(error).toBeCalledTimes(1); + expect(complete).toBeCalledTimes(0); + expect(next.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + Object { + "test": 1, + }, + ] + `); + expect(error.mock.calls[0][0]).toBe(undefined); + }); + }); + + describe('inspector', () => { + let requestResponder: RequestResponder; + beforeEach(() => { + requestResponder = ({ + stats: jest.fn(), + ok: jest.fn(), + error: jest.fn(), + json: jest.fn(), + } as unknown) as RequestResponder; + }); + + test('calls inspector if provided', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource.fetch$(options).toPromise(); + + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.error).not.toBeCalled(); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(1); + // First and last + expect(requestResponder.stats).toBeCalledTimes(2); + }); + + test('calls inspector only once, with multiple subs (shareReplay)', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + const res$ = searchSource.fetch$(options); + + const complete1 = jest.fn(); + const complete2 = jest.fn(); + + res$.subscribe({ + complete: complete1, + }); + res$.subscribe({ + complete: complete2, + }); + + await res$.toPromise(); + + expect(complete1).toBeCalledTimes(1); + expect(complete2).toBeCalledTimes(1); + expect(options.inspector.adapter.start).toBeCalledTimes(1); + }); + + test('calls error on inspector', async () => { + const options = { + inspector: { + title: 'a', + adapter: { + start: jest.fn().mockReturnValue(requestResponder), + } as any, + }, + }; + + searchSourceDependencies.search = jest.fn().mockReturnValue(of(Promise.reject('aaaaa'))); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + await searchSource + .fetch$(options) + .toPromise() + .catch(() => {}); + + expect(options.inspector.adapter.start).toBeCalledTimes(1); + expect(requestResponder.json).toBeCalledTimes(1); + expect(requestResponder.error).toBeCalledTimes(1); + expect(requestResponder.ok).toBeCalledTimes(0); + expect(requestResponder.stats).toBeCalledTimes(0); + }); + }); + + describe('postFlightRequest', () => { + let fetchSub: any; + + function getAggConfigs(typesRegistry: AggTypesRegistryStart, enabled: boolean) { + return new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + } + + beforeEach(() => { + fetchSub = { + next: jest.fn(), + complete: jest.fn(), + error: jest.fn(), + }; + }); + + test('doesnt call any post flight requests if disabled', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn(); + const ac = getAggConfigs(typesRegistry, false); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(2); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(fetchSub.error).toHaveBeenCalledTimes(0); + + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(0); + }); + + test('doesnt call any post flight if searchsource has error', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn(); + const ac = getAggConfigs(typesRegistry, true); + + searchSourceDependencies.search = jest.fn().mockImplementation(() => + of(1).pipe( + switchMap((r) => { + throw r; + }) + ) + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + await fetch$.toPromise().catch((e) => {}); + + expect(fetchSub.next).toHaveBeenCalledTimes(0); + expect(fetchSub.complete).toHaveBeenCalledTimes(0); + expect(fetchSub.error).toHaveBeenNthCalledWith(1, 1); + + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(0); + }); + + test('calls post flight requests, fires 1 extra response, returns last response', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockResolvedValue({ + other: 5, + }); + + const allac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: true, + params: { field: 'field1' }, + }, + { + type: 'avg', + enabled: true, + params: { field: 'field2' }, + }, + { + type: 'avg', + enabled: true, + params: { field: 'foo-bar' }, + }, + ], + { + typesRegistry, + } + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', allac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + const resp = await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(3); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(fetchSub.error).toHaveBeenCalledTimes(0); + expect(resp).toStrictEqual({ other: 5 }); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(3); + }); + + test('calls post flight requests only once, with multiple subs (shareReplay)', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockResolvedValue({ + other: 5, + }); + + const allac = new AggConfigs( + indexPattern3, + [ + { + type: 'avg', + enabled: true, + params: { field: 'field1' }, + }, + ], + { + typesRegistry, + } + ); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', allac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + const fetchSub2 = { + next: jest.fn(), + complete: jest.fn(), + error: jest.fn(), + }; + fetch$.subscribe(fetchSub2); + + await fetch$.toPromise(); + + expect(fetchSub.next).toHaveBeenCalledTimes(3); + expect(fetchSub.complete).toHaveBeenCalledTimes(1); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(1); + }); + + test('calls post flight requests, handles error', async () => { + const typesRegistry = mockAggTypesRegistry(); + typesRegistry.get('avg').postFlightRequest = jest.fn().mockRejectedValue(undefined); + const ac = getAggConfigs(typesRegistry, true); + + searchSource = new SearchSource({}, searchSourceDependencies); + searchSource.setField('index', indexPattern); + searchSource.setField('aggs', ac); + const fetch$ = searchSource.fetch$({}); + fetch$.subscribe(fetchSub); + + await fetch$.toPromise().catch(() => {}); + + expect(fetchSub.next).toHaveBeenCalledTimes(2); + expect(fetchSub.complete).toHaveBeenCalledTimes(0); + expect(fetchSub.error).toHaveBeenCalledTimes(1); + expect(typesRegistry.get('avg').postFlightRequest).toHaveBeenCalledTimes(1); + }); + }); + }); }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index e1e7a8292d677..1c1c32228703f 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -60,12 +60,22 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; -import { catchError, finalize, map, switchMap, tap } from 'rxjs/operators'; -import { defer, from } from 'rxjs'; +import { + catchError, + finalize, + first, + last, + map, + shareReplay, + switchMap, + tap, +} from 'rxjs/operators'; +import { defer, EMPTY, from, Observable } from 'rxjs'; +import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; -import { ISearchGeneric, ISearchOptions } from '../..'; +import { AggConfigs, ISearchGeneric, ISearchOptions } from '../..'; import type { ISearchSource, SearchFieldValue, @@ -75,7 +85,15 @@ import type { import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch'; import { getRequestInspectorStats, getResponseInspectorStats } from './inspect'; -import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common'; +import { + getEsQueryConfig, + buildEsQuery, + Filter, + UI_SETTINGS, + isErrorResponse, + isPartialResponse, + IKibanaSearchResponse, +} from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; import { fetchSoon } from './legacy'; import { extractReferences } from './extract_references'; @@ -256,10 +274,8 @@ export class SearchSource { */ fetch$(options: ISearchOptions = {}) { const { getConfig } = this.dependencies; - return defer(() => this.requestIsStarting(options)).pipe( - tap(() => { - options.requestResponder?.stats(getRequestInspectorStats(this)); - }), + + const s$ = defer(() => this.requestIsStarting(options)).pipe( switchMap(() => { const searchRequest = this.flatten(); this.history = [searchRequest]; @@ -273,21 +289,14 @@ export class SearchSource { }), tap((response) => { // TODO: Remove casting when https://github.com/elastic/elasticsearch-js/issues/1287 is resolved - if ((response as any).error) { + if (!response || (response as any).error) { throw new RequestFailure(null, response); - } else { - options.requestResponder?.stats(getResponseInspectorStats(response, this)); - options.requestResponder?.ok({ json: response }); } }), - catchError((e) => { - options.requestResponder?.error({ json: e }); - throw e; - }), - finalize(() => { - options.requestResponder?.json(this.getSearchRequestBody()); - }) + shareReplay() ); + + return this.inspectSearch(s$, options); } /** @@ -328,9 +337,96 @@ export class SearchSource { * PRIVATE APIS ******/ + private inspectSearch(s$: Observable>, options: ISearchOptions) { + const { id, title, description, adapter } = options.inspector || { title: '' }; + + const requestResponder = adapter?.start(title, { + id, + description, + searchSessionId: options.sessionId, + }); + + const trackRequestBody = () => { + try { + requestResponder?.json(this.getSearchRequestBody()); + } catch (e) {} // eslint-disable-line no-empty + }; + + // Track request stats on first emit, swallow errors + const first$ = s$ + .pipe( + first(undefined, null), + tap(() => { + requestResponder?.stats(getRequestInspectorStats(this)); + trackRequestBody(); + }), + catchError(() => { + trackRequestBody(); + return EMPTY; + }), + finalize(() => { + first$.unsubscribe(); + }) + ) + .subscribe(); + + // Track response stats on last emit, as well as errors + const last$ = s$ + .pipe( + catchError((e) => { + requestResponder?.error({ json: e }); + return EMPTY; + }), + last(undefined, null), + tap((finalResponse) => { + if (finalResponse) { + requestResponder?.stats(getResponseInspectorStats(finalResponse, this)); + requestResponder?.ok({ json: finalResponse }); + } + }), + finalize(() => { + last$.unsubscribe(); + }) + ) + .subscribe(); + + return s$; + } + + private hasPostFlightRequests() { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + return aggs.aggs.some( + (agg) => agg.enabled && typeof agg.type.postFlightRequest === 'function' + ); + } else { + return false; + } + } + + private async fetchOthers(response: estypes.SearchResponse, options: ISearchOptions) { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + for (const agg of aggs.aggs) { + if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { + response = await agg.type.postFlightRequest( + response, + aggs, + agg, + this, + options.inspector?.adapter, + options.abortSignal, + options.sessionId + ); + } + } + return response; + } + } + /** * Run a search using the search service - * @return {Promise>} + * @return {Observable>} */ private fetchSearch$(searchRequest: SearchRequest, options: ISearchOptions) { const { search, getConfig, onResponse } = this.dependencies; @@ -340,6 +436,43 @@ export class SearchSource { }); return search({ params, indexType: searchRequest.indexType }, options).pipe( + switchMap((response) => { + return new Observable>((obs) => { + if (isErrorResponse(response)) { + obs.error(response); + } else if (isPartialResponse(response)) { + obs.next(response); + } else { + if (!this.hasPostFlightRequests()) { + obs.next(response); + obs.complete(); + } else { + // Treat the complete response as partial, then run the postFlightRequests. + obs.next({ + ...response, + isPartial: true, + isRunning: true, + }); + const sub = from(this.fetchOthers(response.rawResponse, options)).subscribe({ + next: (responseWithOther) => { + obs.next({ + ...response, + rawResponse: responseWithOther, + }); + }, + error: (e) => { + obs.error(e); + sub.unsubscribe(); + }, + complete: () => { + obs.complete(); + sub.unsubscribe(); + }, + }); + } + } + }); + }), map(({ rawResponse }) => onResponse(searchRequest, rawResponse)) ); } @@ -452,6 +585,12 @@ export class SearchSource { getConfig(UI_SETTINGS.SORT_OPTIONS) ); return addToBody(key, sort); + case 'aggs': + if ((val as any) instanceof AggConfigs) { + return addToBody('aggs', val.toDsl()); + } else { + return addToBody('aggs', val); + } default: return addToBody(key, val); } diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts index a178b38693d92..99f3f67a5e257 100644 --- a/src/plugins/data/common/search/search_source/types.ts +++ b/src/plugins/data/common/search/search_source/types.ts @@ -7,6 +7,7 @@ */ import { NameList } from 'elasticsearch'; +import { IAggConfigs } from 'src/plugins/data/public'; import { Query } from '../..'; import { Filter } from '../../es_query'; import { IndexPattern } from '../../index_patterns'; @@ -78,7 +79,7 @@ export interface SearchSourceFields { /** * {@link AggConfigs} */ - aggs?: any; + aggs?: object | IAggConfigs | (() => object); from?: number; size?: number; source?: NameList; diff --git a/src/plugins/data/common/search/tabify/index.ts b/src/plugins/data/common/search/tabify/index.ts index 168d4cf9d4c37..74fbc7ba4cfa4 100644 --- a/src/plugins/data/common/search/tabify/index.ts +++ b/src/plugins/data/common/search/tabify/index.ts @@ -6,27 +6,6 @@ * Side Public License, v 1. */ -import { SearchResponse } from 'elasticsearch'; -import { SearchSource } from '../search_source'; -import { tabifyAggResponse } from './tabify'; -import { tabifyDocs, TabifyDocsOptions } from './tabify_docs'; -import { TabbedResponseWriterOptions } from './types'; - -export const tabify = ( - searchSource: SearchSource, - esResponse: SearchResponse, - opts: Partial | TabifyDocsOptions -) => { - return !esResponse.aggregations - ? tabifyDocs(esResponse, searchSource.getField('index'), opts as TabifyDocsOptions) - : tabifyAggResponse( - searchSource.getField('aggs'), - esResponse, - opts as Partial - ); -}; - -export { tabifyDocs }; - +export { tabifyDocs } from './tabify_docs'; export { tabifyAggResponse } from './tabify'; export { tabifyGetColumns } from './get_columns'; diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index 37de8dc49d3c6..e3ec499a0020d 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -9,7 +9,7 @@ import { Observable } from 'rxjs'; import { IEsSearchRequest, IEsSearchResponse } from './es_search'; import { IndexPattern } from '..'; -import type { RequestResponder } from '../../../inspector/common'; +import type { RequestAdapter } from '../../../inspector/common'; export type ISearchGeneric = < SearchStrategyRequest extends IKibanaSearchRequest = IEsSearchRequest, @@ -81,6 +81,13 @@ export interface IKibanaSearchRequest { params?: Params; } +export interface IInspectorInfo { + adapter?: RequestAdapter; + title: string; + id?: string; + description?: string; +} + export interface ISearchOptions { /** * An `AbortSignal` that allows the caller of `search` to abort a search request. @@ -117,10 +124,12 @@ export interface ISearchOptions { /** * Index pattern reference is used for better error messages */ - indexPattern?: IndexPattern; - requestResponder?: RequestResponder; + /** + * Inspector integration options + */ + inspector?: IInspectorInfo; } /** diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 35f13fc855e99..0dd06691d68bb 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -46,6 +46,7 @@ import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_ import { History } from 'history'; import { Href } from 'history'; import { HttpSetup } from 'kibana/public'; +import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { IconType } from '@elastic/eui'; import { IncomingHttpHeaders } from 'http'; import { InjectedIntl } from '@kbn/i18n/react'; @@ -254,6 +255,8 @@ export class AggConfigs { getResponseAggById(id: string): AggConfig | undefined; getResponseAggs(): AggConfig[]; // (undocumented) + hierarchical?: boolean; + // (undocumented) indexPattern: IndexPattern; jsonDataEquals(aggConfigs: AggConfig[]): boolean; // (undocumented) @@ -267,7 +270,7 @@ export class AggConfigs { // (undocumented) timeRange?: TimeRange; // (undocumented) - toDsl(hierarchical?: boolean): Record; + toDsl(): Record; } // @internal (undocumented) @@ -1672,13 +1675,11 @@ export type ISearchGeneric = >; + fetch$(options?: ISearchOptions): Observable>; // @deprecated - fetch(options?: ISearchOptions): Promise>; + fetch(options?: ISearchOptions): Promise>; getField(field: K, recurse?: boolean): SearchSourceFields[K]; getFields(): SearchSourceFields; getId(): string; @@ -2462,7 +2463,7 @@ export class SearchSource { // @public export interface SearchSourceFields { // (undocumented) - aggs?: any; + aggs?: object | IAggConfigs_2 | (() => object); // Warning: (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts fields?: SearchFieldValue[]; // @deprecated diff --git a/src/plugins/data/public/search/expressions/esaggs.test.ts b/src/plugins/data/public/search/expressions/esaggs.test.ts index d7a6446781c43..e75bd7be219de 100644 --- a/src/plugins/data/public/search/expressions/esaggs.test.ts +++ b/src/plugins/data/public/search/expressions/esaggs.test.ts @@ -100,17 +100,20 @@ describe('esaggs expression function - public', () => { expect(handleEsaggsRequest).toHaveBeenCalledWith({ abortSignal: mockHandlers.abortSignal, - aggs: { foo: 'bar' }, + aggs: { + foo: 'bar', + hierarchical: true, + }, filters: undefined, indexPattern: {}, inspectorAdapters: mockHandlers.inspectorAdapters, - metricsAtAllLevels: args.metricsAtAllLevels, partialRows: args.partialRows, query: undefined, searchSessionId: 'abc123', searchSourceService: startDependencies.searchSource, timeFields: args.timeFields, timeRange: undefined, + getNow: undefined, }); }); diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index 45d24af3a6ebb..1e3d56c71e423 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -8,7 +8,6 @@ import { get } from 'lodash'; import { StartServicesAccessor } from 'src/core/public'; -import { Adapters } from 'src/plugins/inspector/common'; import { EsaggsExpressionFunctionDefinition, EsaggsStartDependencies, @@ -44,14 +43,14 @@ export function getFunctionDefinition({ indexPattern, args.aggs!.map((agg) => agg.value) ); + aggConfigs.hierarchical = args.metricsAtAllLevels; return await handleEsaggsRequest({ - abortSignal: (abortSignal as unknown) as AbortSignal, + abortSignal, aggs: aggConfigs, filters: get(input, 'filters', undefined), indexPattern, - inspectorAdapters: inspectorAdapters as Adapters, - metricsAtAllLevels: args.metricsAtAllLevels, + inspectorAdapters, partialRows: args.partialRows, query: get(input, 'query', undefined) as any, searchSessionId: getSearchSessionId(), diff --git a/src/plugins/data/server/search/expressions/esaggs.test.ts b/src/plugins/data/server/search/expressions/esaggs.test.ts index 124a171de6378..15287e9d8cf5b 100644 --- a/src/plugins/data/server/search/expressions/esaggs.test.ts +++ b/src/plugins/data/server/search/expressions/esaggs.test.ts @@ -108,11 +108,13 @@ describe('esaggs expression function - server', () => { expect(handleEsaggsRequest).toHaveBeenCalledWith({ abortSignal: mockHandlers.abortSignal, - aggs: { foo: 'bar' }, + aggs: { + foo: 'bar', + hierarchical: args.metricsAtAllLevels, + }, filters: undefined, indexPattern: {}, inspectorAdapters: mockHandlers.inspectorAdapters, - metricsAtAllLevels: args.metricsAtAllLevels, partialRows: args.partialRows, query: undefined, searchSessionId: 'abc123', diff --git a/src/plugins/data/server/search/expressions/esaggs.ts b/src/plugins/data/server/search/expressions/esaggs.ts index 61fd320d89b95..bb22a491b157e 100644 --- a/src/plugins/data/server/search/expressions/esaggs.ts +++ b/src/plugins/data/server/search/expressions/esaggs.ts @@ -9,7 +9,6 @@ import { get } from 'lodash'; import { i18n } from '@kbn/i18n'; import { KibanaRequest, StartServicesAccessor } from 'src/core/server'; -import { Adapters } from 'src/plugins/inspector/common'; import { EsaggsExpressionFunctionDefinition, EsaggsStartDependencies, @@ -61,13 +60,14 @@ export function getFunctionDefinition({ args.aggs!.map((agg) => agg.value) ); + aggConfigs.hierarchical = args.metricsAtAllLevels; + return await handleEsaggsRequest({ - abortSignal: (abortSignal as unknown) as AbortSignal, + abortSignal, aggs: aggConfigs, filters: get(input, 'filters', undefined), indexPattern, - inspectorAdapters: inspectorAdapters as Adapters, - metricsAtAllLevels: args.metricsAtAllLevels, + inspectorAdapters, partialRows: args.partialRows, query: get(input, 'query', undefined) as any, searchSessionId: getSearchSessionId(), diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 622356c4441ac..3316e8102e50a 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -26,12 +26,14 @@ import { Ensure } from '@kbn/utility-types'; import { EnvironmentMode } from '@kbn/config'; import { ErrorToastOptions } from 'src/core/public/notifications'; import { estypes } from '@elastic/elasticsearch'; +import { EventEmitter } from 'events'; import { ExecutionContext } from 'src/plugins/expressions/common'; import { ExpressionAstExpression } from 'src/plugins/expressions/common'; import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common'; import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ExpressionValueBoxed } from 'src/plugins/expressions/common'; import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_formats/utils'; +import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { ISavedObjectsRepository } from 'src/core/server'; import { IScopedClusterClient } from 'src/core/server'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; @@ -999,13 +1001,11 @@ export interface IScopedSearchClient extends ISearchClient { export interface ISearchOptions { abortSignal?: AbortSignal; indexPattern?: IndexPattern; + // Warning: (ae-forgotten-export) The symbol "IInspectorInfo" needs to be exported by the entry point index.d.ts + inspector?: IInspectorInfo; isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; - // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts - // - // (undocumented) - requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 35a89eb45f35e..4099d5e8ef7e2 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -415,11 +415,20 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.LOADING; $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); + inspectorAdapters.requests.reset(); return $scope.volatileSearchSource .fetch$({ abortSignal: abortController.signal, sessionId: searchSessionId, - requestResponder: getRequestResponder({ searchSessionId }), + inspector: { + adapter: inspectorAdapters.requests, + title: i18n.translate('discover.inspectorRequestDataTitle', { + defaultMessage: 'data', + }), + description: i18n.translate('discover.inspectorRequestDescription', { + defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', + }), + }, }) .toPromise() .then(onResults) @@ -465,17 +474,6 @@ function discoverController($route, $scope) { await refetch$.next(); }; - function getRequestResponder({ searchSessionId = null } = { searchSessionId: null }) { - inspectorAdapters.requests.reset(); - const title = i18n.translate('discover.inspectorRequestDataTitle', { - defaultMessage: 'data', - }); - const description = i18n.translate('discover.inspectorRequestDescription', { - defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', - }); - return inspectorAdapters.requests.start(title, { description, searchSessionId }); - } - $scope.resetQuery = function () { history.push( $route.current.params.id ? `/view/${encodeURIComponent($route.current.params.id)}` : '/' diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index 237da72ae3a52..dbaf07fed18c2 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -317,17 +317,6 @@ export class SearchEmbeddable // Log request to inspector this.inspectorAdapters.requests!.reset(); - const title = i18n.translate('discover.embeddable.inspectorRequestDataTitle', { - defaultMessage: 'Data', - }); - const description = i18n.translate('discover.embeddable.inspectorRequestDescription', { - defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', - }); - - const requestResponder = this.inspectorAdapters.requests!.start(title, { - description, - searchSessionId, - }); this.searchScope.$apply(() => { this.searchScope!.isLoading = true; @@ -340,7 +329,16 @@ export class SearchEmbeddable .fetch$({ abortSignal: this.abortController.signal, sessionId: searchSessionId, - requestResponder, + inspector: { + adapter: this.inspectorAdapters.requests, + title: i18n.translate('discover.embeddable.inspectorRequestDataTitle', { + defaultMessage: 'Data', + }), + description: i18n.translate('discover.embeddable.inspectorRequestDescription', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the search.', + }), + }, }) .toPromise(); this.updateOutput({ loading: false, error: undefined }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts index 2915eaec8ac77..50043772af95b 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts @@ -167,12 +167,6 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource const abortController = new AbortController(); registerCancelCallback(() => abortController.abort()); - const requestResponder = this.getInspectorAdapters()?.requests?.start(requestName, { - id: requestId, - description: requestDescription, - searchSessionId, - }); - let resp; try { resp = await searchSource @@ -180,7 +174,12 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource abortSignal: abortController.signal, sessionId: searchSessionId, legacyHitsTotal: false, - requestResponder, + inspector: { + adapter: this.getInspectorAdapters()?.requests, + id: requestId, + title: requestName, + description: requestDescription, + }, }) .toPromise(); } catch (error) { diff --git a/x-pack/test/examples/search_examples/index.ts b/x-pack/test/examples/search_examples/index.ts index 13eac7566525e..65e214cda4cf8 100644 --- a/x-pack/test/examples/search_examples/index.ts +++ b/x-pack/test/examples/search_examples/index.ts @@ -23,7 +23,8 @@ export default function ({ getService, loadTestFile }: PluginFunctionalProviderC await esArchiver.unload('lens/basic'); }); - loadTestFile(require.resolve('./search_sessions_cache')); loadTestFile(require.resolve('./search_session_example')); + loadTestFile(require.resolve('./search_example')); + loadTestFile(require.resolve('./search_sessions_cache')); }); } diff --git a/x-pack/test/examples/search_examples/search_example.ts b/x-pack/test/examples/search_examples/search_example.ts new file mode 100644 index 0000000000000..19a9535ebb951 --- /dev/null +++ b/x-pack/test/examples/search_examples/search_example.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../functional/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + const PageObjects = getPageObjects(['common', 'timePicker']); + const retry = getService('retry'); + + describe.skip('Search session example', () => { + const appId = 'searchExamples'; + + before(async function () { + await PageObjects.common.navigateToApp(appId, { insertTimestamp: false }); + }); + + it('should have an other bucket', async () => { + await PageObjects.timePicker.setAbsoluteRange( + 'Jan 1, 2014 @ 00:00:00.000', + 'Jan 1, 2016 @ 00:00:00.000' + ); + await testSubjects.click('searchSourceWithOther'); + + await retry.waitFor('has other bucket', async () => { + await testSubjects.click('responseTab'); + const codeBlock = await testSubjects.find('responseCodeBlock'); + const visibleText = await codeBlock.getVisibleText(); + return visibleText.indexOf('__other__') > -1; + }); + }); + }); +} From fed17c2b6e71deefb4ff30a1eabff1cb485de283 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 18 Apr 2021 16:40:54 +0200 Subject: [PATCH 21/25] Rule registry bundle size (#97251) --- x-pack/plugins/apm/common/alert_types.ts | 6 +-- .../plugins/apm/common/anomaly_detection.ts | 2 +- x-pack/plugins/apm/common/ml_constants.ts | 24 ++++++++++++ x-pack/plugins/apm/common/rules.ts | 25 ------------ .../apm/common/rules/apm_rule_field_map.ts | 20 ++++++++++ .../rules/apm_rule_registry_settings.ts | 10 +++++ .../apm/common/service_health_status.ts | 2 +- .../alerting/register_apm_alerts.ts | 19 +++++++--- .../index.tsx | 2 +- .../select_anomaly_severity.test.tsx | 2 +- x-pack/plugins/apm/public/plugin.ts | 38 ++++++++++--------- ...action_duration_anomaly_alert_type.test.ts | 2 +- ...transaction_duration_anomaly_alert_type.ts | 2 +- .../transactions/get_anomaly_data/index.ts | 2 +- x-pack/plugins/apm/server/plugin.ts | 10 +++-- .../common/observability_rule_registry.ts | 22 ----------- .../rules/observability_rule_field_map.ts | 22 +++++++++++ .../observability_rule_registry_settings.ts | 10 +++++ .../public/pages/alerts/index.tsx | 3 +- x-pack/plugins/observability/public/plugin.ts | 24 ++++++------ .../public/rules/formatter_rule_registry.ts | 5 +++ x-pack/plugins/observability/server/plugin.ts | 10 +++-- x-pack/plugins/rule_registry/kibana.json | 5 +-- x-pack/plugins/rule_registry/public/index.ts | 4 +- x-pack/plugins/rule_registry/public/plugin.ts | 4 +- .../public/rule_registry/types.ts | 4 +- 26 files changed, 169 insertions(+), 110 deletions(-) create mode 100644 x-pack/plugins/apm/common/ml_constants.ts delete mode 100644 x-pack/plugins/apm/common/rules.ts create mode 100644 x-pack/plugins/apm/common/rules/apm_rule_field_map.ts create mode 100644 x-pack/plugins/apm/common/rules/apm_rule_registry_settings.ts delete mode 100644 x-pack/plugins/observability/common/observability_rule_registry.ts create mode 100644 x-pack/plugins/observability/common/rules/observability_rule_field_map.ts create mode 100644 x-pack/plugins/observability/common/rules/observability_rule_registry_settings.ts diff --git a/x-pack/plugins/apm/common/alert_types.ts b/x-pack/plugins/apm/common/alert_types.ts index 62bd07ce6f500..12df93d54b296 100644 --- a/x-pack/plugins/apm/common/alert_types.ts +++ b/x-pack/plugins/apm/common/alert_types.ts @@ -6,9 +6,9 @@ */ import { i18n } from '@kbn/i18n'; -import { ValuesType } from 'utility-types'; -import { ActionGroup } from '../../alerting/common'; -import { ANOMALY_SEVERITY, ANOMALY_THRESHOLD } from '../../ml/common'; +import type { ValuesType } from 'utility-types'; +import type { ActionGroup } from '../../alerting/common'; +import { ANOMALY_SEVERITY, ANOMALY_THRESHOLD } from './ml_constants'; export enum AlertType { ErrorCount = 'apm.error_rate', // ErrorRate was renamed to ErrorCount but the key is kept as `error_rate` for backwards-compat. diff --git a/x-pack/plugins/apm/common/anomaly_detection.ts b/x-pack/plugins/apm/common/anomaly_detection.ts index b9cc3de8bb5d0..43a779407d2a4 100644 --- a/x-pack/plugins/apm/common/anomaly_detection.ts +++ b/x-pack/plugins/apm/common/anomaly_detection.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { ANOMALY_SEVERITY } from '../../ml/common'; +import { ANOMALY_SEVERITY } from './ml_constants'; import { getSeverityType, getSeverityColor as mlGetSeverityColor, diff --git a/x-pack/plugins/apm/common/ml_constants.ts b/x-pack/plugins/apm/common/ml_constants.ts new file mode 100644 index 0000000000000..7818299d9d883 --- /dev/null +++ b/x-pack/plugins/apm/common/ml_constants.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// copied from ml/common, to keep the bundle size small +export enum ANOMALY_SEVERITY { + CRITICAL = 'critical', + MAJOR = 'major', + MINOR = 'minor', + WARNING = 'warning', + LOW = 'low', + UNKNOWN = 'unknown', +} + +export enum ANOMALY_THRESHOLD { + CRITICAL = 75, + MAJOR = 50, + MINOR = 25, + WARNING = 3, + LOW = 0, +} diff --git a/x-pack/plugins/apm/common/rules.ts b/x-pack/plugins/apm/common/rules.ts deleted file mode 100644 index a3b60a785f5c7..0000000000000 --- a/x-pack/plugins/apm/common/rules.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -const plainApmRuleRegistrySettings = { - name: 'apm', - fieldMap: { - 'service.environment': { - type: 'keyword', - }, - 'transaction.type': { - type: 'keyword', - }, - 'processor.event': { - type: 'keyword', - }, - }, -} as const; - -type APMRuleRegistrySettings = typeof plainApmRuleRegistrySettings; - -export const apmRuleRegistrySettings: APMRuleRegistrySettings = plainApmRuleRegistrySettings; diff --git a/x-pack/plugins/apm/common/rules/apm_rule_field_map.ts b/x-pack/plugins/apm/common/rules/apm_rule_field_map.ts new file mode 100644 index 0000000000000..9bbd9381c2319 --- /dev/null +++ b/x-pack/plugins/apm/common/rules/apm_rule_field_map.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const apmRuleFieldMap = { + 'service.environment': { + type: 'keyword', + }, + 'transaction.type': { + type: 'keyword', + }, + 'processor.event': { + type: 'keyword', + }, +} as const; + +export type APMRuleFieldMap = typeof apmRuleFieldMap; diff --git a/x-pack/plugins/apm/common/rules/apm_rule_registry_settings.ts b/x-pack/plugins/apm/common/rules/apm_rule_registry_settings.ts new file mode 100644 index 0000000000000..1257db4e6a4d3 --- /dev/null +++ b/x-pack/plugins/apm/common/rules/apm_rule_registry_settings.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const apmRuleRegistrySettings = { + name: 'apm', +}; diff --git a/x-pack/plugins/apm/common/service_health_status.ts b/x-pack/plugins/apm/common/service_health_status.ts index 71c373a48c9d5..b5318f9333e4f 100644 --- a/x-pack/plugins/apm/common/service_health_status.ts +++ b/x-pack/plugins/apm/common/service_health_status.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import { EuiTheme } from '../../../../src/plugins/kibana_react/common'; -import { ANOMALY_SEVERITY } from '../../ml/common'; +import { ANOMALY_SEVERITY } from './ml_constants'; export enum ServiceHealthStatus { healthy = 'healthy', diff --git a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts index 8834cbc70e0b1..583be94c30a34 100644 --- a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts +++ b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts @@ -7,11 +7,20 @@ import { i18n } from '@kbn/i18n'; import { lazy } from 'react'; -import { format } from 'url'; +import { stringify } from 'querystring'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; -import { asDuration, asPercent } from '../../../common/utils/formatters'; import { AlertType } from '../../../common/alert_types'; -import { ApmRuleRegistry } from '../../plugin'; +import type { ApmRuleRegistry } from '../../plugin'; + +const format = ({ + pathname, + query, +}: { + pathname: string; + query: Record; +}): string => { + return `${pathname}?${stringify(query)}`; +}; export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { apmRuleRegistry.registerType({ @@ -71,7 +80,7 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { 'Alert when the latency of a specific transaction type in a service exceeds a defined threshold.', } ), - format: ({ alert }) => ({ + format: ({ alert, formatters: { asDuration } }) => ({ reason: i18n.translate( 'xpack.apm.alertTypes.transactionDuration.reason', { @@ -131,7 +140,7 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { 'Alert when the rate of transaction errors in a service exceeds a defined threshold.', } ), - format: ({ alert }) => ({ + format: ({ alert, formatters: { asPercent } }) => ({ reason: i18n.translate( 'xpack.apm.alertTypes.transactionErrorRate.reason', { diff --git a/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/index.tsx index 62926796cafb4..10d139f6ccea3 100644 --- a/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/index.tsx @@ -8,7 +8,7 @@ import { useParams } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; import React from 'react'; -import { ANOMALY_SEVERITY } from '../../../../../ml/common'; +import { ANOMALY_SEVERITY } from '../../../../common/ml_constants'; import { useEnvironmentsFetcher } from '../../../hooks/use_environments_fetcher'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; import { ServiceAlertTrigger } from '../service_alert_trigger'; diff --git a/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/select_anomaly_severity.test.tsx b/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/select_anomaly_severity.test.tsx index 85f48ae151e10..7b56eaa4721de 100644 --- a/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/select_anomaly_severity.test.tsx +++ b/x-pack/plugins/apm/public/components/alerting/transaction_duration_anomaly_alert_trigger/select_anomaly_severity.test.tsx @@ -8,7 +8,7 @@ import { render } from '@testing-library/react'; import React, { ReactNode } from 'react'; import { IntlProvider } from 'react-intl'; -import { ANOMALY_SEVERITY } from '../../../../../ml/common'; +import { ANOMALY_SEVERITY } from '../../../../common/ml_constants'; import { SelectAnomalySeverity } from './select_anomaly_severity'; function Wrapper({ children }: { children?: ReactNode }) { diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index 391c54c1e2497..143076e56c831 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -5,13 +5,7 @@ * 2.0. */ -import { ConfigSchema } from '.'; -import { - FetchDataParams, - FormatterRuleRegistry, - HasDataParams, - ObservabilityPublicSetup, -} from '../../observability/public'; +import type { ConfigSchema } from '.'; import { AppMountParameters, CoreSetup, @@ -20,28 +14,35 @@ import { Plugin, PluginInitializerContext, } from '../../../../src/core/public'; -import { +import type { DataPublicPluginSetup, DataPublicPluginStart, } from '../../../../src/plugins/data/public'; -import { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; -import { +import type { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; +import type { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; +import type { PluginSetupContract as AlertingPluginPublicSetup, PluginStartContract as AlertingPluginPublicStart, } from '../../alerting/public'; -import { FeaturesPluginSetup } from '../../features/public'; -import { LicensingPluginSetup } from '../../licensing/public'; -import { +import type { FeaturesPluginSetup } from '../../features/public'; +import type { LicensingPluginSetup } from '../../licensing/public'; +import type { MapsStartApi } from '../../maps/public'; +import type { MlPluginSetup, MlPluginStart } from '../../ml/public'; +import type { + FetchDataParams, + HasDataParams, + ObservabilityPublicSetup, +} from '../../observability/public'; +import { FormatterRuleRegistry } from '../../observability/public'; +import type { TriggersAndActionsUIPublicPluginSetup, TriggersAndActionsUIPublicPluginStart, } from '../../triggers_actions_ui/public'; +import { apmRuleRegistrySettings } from '../common/rules/apm_rule_registry_settings'; +import type { APMRuleFieldMap } from '../common/rules/apm_rule_field_map'; +import { registerApmAlerts } from './components/alerting/register_apm_alerts'; import { featureCatalogueEntry } from './featureCatalogueEntry'; import { toggleAppLinkInNav } from './toggleAppLinkInNav'; -import { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; -import { registerApmAlerts } from './components/alerting/register_apm_alerts'; -import { MlPluginSetup, MlPluginStart } from '../../ml/public'; -import { MapsStartApi } from '../../maps/public'; -import { apmRuleRegistrySettings } from '../common/rules'; export type ApmPluginSetup = ReturnType; export type ApmRuleRegistry = ApmPluginSetup['ruleRegistry']; @@ -162,6 +163,7 @@ export class ApmPlugin implements Plugin { const apmRuleRegistry = plugins.observability.ruleRegistry.create({ ...apmRuleRegistrySettings, + fieldMap: {} as APMRuleFieldMap, ctor: FormatterRuleRegistry, }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts index b9346b2bf4649..ad1a8fcbf6e55 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -5,7 +5,7 @@ * 2.0. */ import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; -import { ANOMALY_SEVERITY } from '../../../../ml/common'; +import { ANOMALY_SEVERITY } from '../../../common/ml_constants'; import { Job, MlPluginSetup } from '../../../../ml/server'; import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; import { createRuleTypeMocks } from './test_utils'; diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 66eb7125b0370..67ff7cdb8e4e0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -18,7 +18,7 @@ import { TRANSACTION_TYPE, } from '../../../common/elasticsearch_fieldnames'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; -import { ANOMALY_SEVERITY } from '../../../../ml/common'; +import { ANOMALY_SEVERITY } from '../../../common/ml_constants'; import { KibanaRequest } from '../../../../../../src/core/server'; import { AlertType, diff --git a/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/index.ts index a03b1ac82e90a..bcd279c57f4a5 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_anomaly_data/index.ts @@ -14,7 +14,7 @@ import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { anomalySeriesFetcher } from './fetcher'; import { getMLJobIds } from '../../service_map/get_service_anomalies'; -import { ANOMALY_THRESHOLD } from '../../../../../ml/common'; +import { ANOMALY_THRESHOLD } from '../../../../common/ml_constants'; import { withApmSpan } from '../../../utils/with_apm_span'; export async function getAnomalySeries({ diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 714b887a4008b..d62a3e6a5d5d7 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -42,7 +42,8 @@ import { } from './types'; import { registerRoutes } from './routes/register_routes'; import { getGlobalApmServerRouteRepository } from './routes/get_global_apm_server_route_repository'; -import { apmRuleRegistrySettings } from '../common/rules'; +import { apmRuleRegistrySettings } from '../common/rules/apm_rule_registry_settings'; +import { apmRuleFieldMap } from '../common/rules/apm_rule_field_map'; export type APMRuleRegistry = ReturnType['ruleRegistry']; @@ -151,9 +152,10 @@ export class APMPlugin config: await mergedConfig$.pipe(take(1)).toPromise(), }); - const apmRuleRegistry = plugins.observability.ruleRegistry.create( - apmRuleRegistrySettings - ); + const apmRuleRegistry = plugins.observability.ruleRegistry.create({ + ...apmRuleRegistrySettings, + fieldMap: apmRuleFieldMap, + }); registerApmAlerts({ registry: apmRuleRegistry, diff --git a/x-pack/plugins/observability/common/observability_rule_registry.ts b/x-pack/plugins/observability/common/observability_rule_registry.ts deleted file mode 100644 index 9254401fc19c4..0000000000000 --- a/x-pack/plugins/observability/common/observability_rule_registry.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { ecsFieldMap, pickWithPatterns } from '../../rule_registry/common'; - -export const observabilityRuleRegistrySettings = { - name: 'observability', - fieldMap: { - ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), - 'kibana.observability.evaluation.value': { - type: 'scaled_float' as const, - scaling_factor: 1000, - }, - 'kibana.observability.evaluation.threshold': { - type: 'scaled_float' as const, - scaling_factor: 1000, - }, - }, -}; diff --git a/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts b/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts new file mode 100644 index 0000000000000..370f5d4ef79f2 --- /dev/null +++ b/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ecsFieldMap, pickWithPatterns } from '../../../rule_registry/common'; + +export const observabilityRuleFieldMap = { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + 'kibana.observability.evaluation.value': { + type: 'scaled_float' as const, + scaling_factor: 1000, + }, + 'kibana.observability.evaluation.threshold': { + type: 'scaled_float' as const, + scaling_factor: 1000, + }, +}; + +export type ObservabilityRuleFieldMap = typeof observabilityRuleFieldMap; diff --git a/x-pack/plugins/observability/common/rules/observability_rule_registry_settings.ts b/x-pack/plugins/observability/common/rules/observability_rule_registry_settings.ts new file mode 100644 index 0000000000000..c901d912eb70f --- /dev/null +++ b/x-pack/plugins/observability/common/rules/observability_rule_registry_settings.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const observabilityRuleRegistrySettings = { + name: 'observability', +}; diff --git a/x-pack/plugins/observability/public/pages/alerts/index.tsx b/x-pack/plugins/observability/public/pages/alerts/index.tsx index 0089465003393..aa5fb2c32ea11 100644 --- a/x-pack/plugins/observability/public/pages/alerts/index.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/index.tsx @@ -24,6 +24,7 @@ import { usePluginContext } from '../../hooks/use_plugin_context'; import { RouteParams } from '../../routes'; import { callObservabilityApi } from '../../services/call_observability_api'; import { getAbsoluteDateRange } from '../../utils/date'; +import { asDuration, asPercent } from '../../../common/utils/formatters'; import { AlertsSearchBar } from './alerts_search_bar'; import { AlertsTable } from './alerts_table'; @@ -68,7 +69,7 @@ export function AlertsPage({ routeParams }: AlertsPageProps) { const formatted = { link: undefined, reason: alert['rule.name'], - ...(ruleType?.format?.({ alert }) ?? {}), + ...(ruleType?.format?.({ alert, formatters: { asDuration, asPercent } }) ?? {}), }; const parsedLink = formatted.link ? parse(formatted.link, true) : undefined; diff --git a/x-pack/plugins/observability/public/plugin.ts b/x-pack/plugins/observability/public/plugin.ts index 491eb36d01ac0..1f56bdebbbb9b 100644 --- a/x-pack/plugins/observability/public/plugin.ts +++ b/x-pack/plugins/observability/public/plugin.ts @@ -5,32 +5,33 @@ * 2.0. */ -import { BehaviorSubject } from 'rxjs'; import { i18n } from '@kbn/i18n'; -import type { RuleRegistryPublicPluginSetupContract } from '../../rule_registry/public'; -import type { - DataPublicPluginSetup, - DataPublicPluginStart, -} from '../../../../src/plugins/data/public'; +import { BehaviorSubject } from 'rxjs'; import { AppMountParameters, AppUpdater, CoreSetup, + CoreStart, DEFAULT_APP_CATEGORIES, Plugin as PluginClass, PluginInitializerContext, - CoreStart, } from '../../../../src/core/public'; +import type { + DataPublicPluginSetup, + DataPublicPluginStart, +} from '../../../../src/plugins/data/public'; import type { HomePublicPluginSetup, HomePublicPluginStart, } from '../../../../src/plugins/home/public'; -import { registerDataHandler } from './data_handler'; -import { toggleOverviewLinkInNav } from './toggle_overview_link_in_nav'; import type { LensPublicStart } from '../../lens/public'; -import { createCallObservabilityApi } from './services/call_observability_api'; -import { observabilityRuleRegistrySettings } from '../common/observability_rule_registry'; +import type { RuleRegistryPublicPluginSetupContract } from '../../rule_registry/public'; +import type { ObservabilityRuleFieldMap } from '../common/rules/observability_rule_field_map'; +import { observabilityRuleRegistrySettings } from '../common/rules/observability_rule_registry_settings'; +import { registerDataHandler } from './data_handler'; import { FormatterRuleRegistry } from './rules/formatter_rule_registry'; +import { createCallObservabilityApi } from './services/call_observability_api'; +import { toggleOverviewLinkInNav } from './toggle_overview_link_in_nav'; export type ObservabilityPublicSetup = ReturnType; export type ObservabilityRuleRegistry = ObservabilityPublicSetup['ruleRegistry']; @@ -72,6 +73,7 @@ export class Plugin const observabilityRuleRegistry = pluginsSetup.ruleRegistry.registry.create({ ...observabilityRuleRegistrySettings, + fieldMap: {} as ObservabilityRuleFieldMap, ctor: FormatterRuleRegistry, }); diff --git a/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts b/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts index 87e6b3c324634..0d0d22cf750fb 100644 --- a/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts +++ b/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts @@ -7,12 +7,17 @@ import type { RuleType } from '../../../rule_registry/public'; import type { BaseRuleFieldMap, OutputOfFieldMap } from '../../../rule_registry/common'; import { RuleRegistry } from '../../../rule_registry/public'; +import type { asDuration, asPercent } from '../../common/utils/formatters'; type AlertTypeOf = OutputOfFieldMap; type FormattableRuleType = RuleType & { format?: (options: { alert: AlertTypeOf; + formatters: { + asDuration: typeof asDuration; + asPercent: typeof asPercent; + }; }) => { reason?: string; link?: string; diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index b167600e788a4..b5208260297d0 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -16,7 +16,8 @@ import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server import { uiSettings } from './ui_settings'; import { registerRoutes } from './routes/register_routes'; import { getGlobalObservabilityServerRouteRepository } from './routes/get_global_observability_server_route_repository'; -import { observabilityRuleRegistrySettings } from '../common/observability_rule_registry'; +import { observabilityRuleRegistrySettings } from '../common/rules/observability_rule_registry_settings'; +import { observabilityRuleFieldMap } from '../common/rules/observability_rule_field_map'; export type ObservabilityPluginSetup = ReturnType; export type ObservabilityRuleRegistry = ObservabilityPluginSetup['ruleRegistry']; @@ -50,9 +51,10 @@ export class ObservabilityPlugin implements Plugin { }); } - const observabilityRuleRegistry = plugins.ruleRegistry.create( - observabilityRuleRegistrySettings - ); + const observabilityRuleRegistry = plugins.ruleRegistry.create({ + ...observabilityRuleRegistrySettings, + fieldMap: observabilityRuleFieldMap, + }); registerRoutes({ core: { diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json index 1636f88a21a61..ec2b366f739e6 100644 --- a/x-pack/plugins/rule_registry/kibana.json +++ b/x-pack/plugins/rule_registry/kibana.json @@ -11,8 +11,5 @@ "triggersActionsUi" ], "server": true, - "ui": true, - "extraPublicDirs": [ - "common" - ] + "ui": true } diff --git a/x-pack/plugins/rule_registry/public/index.ts b/x-pack/plugins/rule_registry/public/index.ts index 55662dbcc8bfc..59697261ff20b 100644 --- a/x-pack/plugins/rule_registry/public/index.ts +++ b/x-pack/plugins/rule_registry/public/index.ts @@ -5,10 +5,10 @@ * 2.0. */ -import { PluginInitializerContext } from 'kibana/public'; +import type { PluginInitializerContext } from 'kibana/public'; import { Plugin } from './plugin'; -export { RuleRegistryPublicPluginSetupContract } from './plugin'; +export type { RuleRegistryPublicPluginSetupContract } from './plugin'; export { RuleRegistry } from './rule_registry'; export type { IRuleRegistry, RuleType } from './rule_registry/types'; diff --git a/x-pack/plugins/rule_registry/public/plugin.ts b/x-pack/plugins/rule_registry/public/plugin.ts index 66c9a4fa224a5..7f0bceefb6797 100644 --- a/x-pack/plugins/rule_registry/public/plugin.ts +++ b/x-pack/plugins/rule_registry/public/plugin.ts @@ -19,7 +19,7 @@ import type { TriggersAndActionsUIPublicPluginSetup, TriggersAndActionsUIPublicPluginStart, } from '../../triggers_actions_ui/public'; -import { baseRuleFieldMap } from '../common'; +import type { BaseRuleFieldMap } from '../common'; import { RuleRegistry } from './rule_registry'; interface RuleRegistrySetupPlugins { @@ -40,7 +40,7 @@ export class Plugin public setup(core: CoreSetup, plugins: RuleRegistrySetupPlugins) { const rootRegistry = new RuleRegistry({ - fieldMap: baseRuleFieldMap, + fieldMap: {} as BaseRuleFieldMap, alertTypeRegistry: plugins.triggersActionsUi.alertTypeRegistry, }); return { diff --git a/x-pack/plugins/rule_registry/public/rule_registry/types.ts b/x-pack/plugins/rule_registry/public/rule_registry/types.ts index bb16227cbab5f..7c186385ebd35 100644 --- a/x-pack/plugins/rule_registry/public/rule_registry/types.ts +++ b/x-pack/plugins/rule_registry/public/rule_registry/types.ts @@ -4,8 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { AlertTypeRegistryContract } from '../../../triggers_actions_ui/public'; -import { BaseRuleFieldMap, FieldMap } from '../../common'; +import type { AlertTypeRegistryContract } from '../../../triggers_actions_ui/public'; +import type { BaseRuleFieldMap, FieldMap } from '../../common'; export interface RuleRegistryConstructorOptions { fieldMap: TFieldMap; From 05bd1c0cdbed2f2b5586af517a5c2d42ae5a366b Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Sun, 18 Apr 2021 16:47:24 +0200 Subject: [PATCH 22/25] [Fleet] Finer-grained error information from install/upgrade API (#95649) * Intercept installation errors and add meta info. * Adjust mock. * Catch errors in all steps of install/upgrade. * Adjust handler for direct package upload. * Don't throw not-found errors on assets during rollback. * Correctly catch errors from _installPackage() * Propagate error from installResult in bulk install case. * Add tests for rollback. * Remove unused code. * Skipping test that doesn't test what it says. * Fix and reenable test. --- .../plugins/fleet/common/types/models/epm.ts | 2 +- .../fleet/common/types/rest_spec/epm.ts | 7 +- .../fleet/server/routes/epm/handlers.ts | 46 ++-- .../epm/packages/bulk_install_packages.ts | 50 +++-- .../ensure_installed_default_packages.test.ts | 10 +- .../server/services/epm/packages/install.ts | 208 ++++++++++-------- .../server/services/epm/packages/remove.ts | 7 +- .../fleet_api_integration/apis/epm/index.js | 1 + .../apis/epm/install_error_rollback.ts | 61 +++++ .../error_handling/0.1.0/docs/README.md | 3 + .../visualization/sample_visualization.json | 14 ++ .../error_handling/0.1.0/manifest.yml | 20 ++ .../error_handling/0.2.0/docs/README.md | 5 + .../visualization/sample_visualization.json | 14 ++ .../error_handling/0.2.0/manifest.yml | 19 ++ 15 files changed, 327 insertions(+), 140 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/kibana/visualization/sample_visualization.json create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/kibana/visualization/sample_visualization.json create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 3bc0d97d64646..1a594e77f4857 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -30,7 +30,7 @@ export enum InstallStatus { uninstalling = 'uninstalling', } -export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install'; +export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown'; export type InstallSource = 'registry' | 'upload'; export type EpmPackageInstallStatus = 'installed' | 'installing'; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 3c7a32265d20a..e5c7ace420c73 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -12,6 +12,7 @@ import type { RegistrySearchResult, PackageInfo, PackageUsageStats, + InstallType, } from '../models/epm'; export interface GetCategoriesRequest { @@ -83,8 +84,10 @@ export interface IBulkInstallPackageHTTPError { } export interface InstallResult { - assets: AssetReference[]; - status: 'installed' | 'already_installed'; + assets?: AssetReference[]; + status?: 'installed' | 'already_installed'; + error?: Error; + installType: InstallType; } export interface BulkInstallPackageInfo { diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index f0d6e68427361..16d583f8a8d1f 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -226,20 +226,21 @@ export const installPackageFromRegistryHandler: RequestHandler< const savedObjectsClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asCurrentUser; const { pkgkey } = request.params; - try { - const res = await installPackage({ - installSource: 'registry', - savedObjectsClient, - pkgkey, - esClient, - force: request.body?.force, - }); + + const res = await installPackage({ + installSource: 'registry', + savedObjectsClient, + pkgkey, + esClient, + force: request.body?.force, + }); + if (!res.error) { const body: InstallPackageResponse = { - response: res.assets, + response: res.assets || [], }; return response.ok({ body }); - } catch (e) { - return await defaultIngestErrorHandler({ error: e, response }); + } else { + return await defaultIngestErrorHandler({ error: res.error, response }); } }; @@ -292,20 +293,21 @@ export const installPackageByUploadHandler: RequestHandler< const esClient = context.core.elasticsearch.client.asCurrentUser; const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later const archiveBuffer = Buffer.from(request.body); - try { - const res = await installPackage({ - installSource: 'upload', - savedObjectsClient, - esClient, - archiveBuffer, - contentType, - }); + + const res = await installPackage({ + installSource: 'upload', + savedObjectsClient, + esClient, + archiveBuffer, + contentType, + }); + if (!res.error) { const body: InstallPackageResponse = { - response: res.assets, + response: res.assets || [], }; return response.ok({ body }); - } catch (error) { - return defaultIngestErrorHandler({ error, response }); + } else { + return defaultIngestErrorHandler({ error: res.error, response }); } }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index 7323263d4a70f..baaaaf6c6b0cf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -32,22 +32,27 @@ export async function bulkInstallPackages({ ); logger.debug(`kicking off bulk install of ${packagesToInstall.join(', ')} from registry`); - const installResults = await Promise.allSettled( + const bulkInstallResults = await Promise.allSettled( latestPackagesResults.map(async (result, index) => { const packageName = packagesToInstall[index]; if (result.status === 'fulfilled') { const latestPackage = result.value; - return { - name: packageName, - version: latestPackage.version, - result: await installPackage({ - savedObjectsClient, - esClient, - pkgkey: Registry.pkgToPkgKey(latestPackage), - installSource, - skipPostInstall: true, - }), - }; + const installResult = await installPackage({ + savedObjectsClient, + esClient, + pkgkey: Registry.pkgToPkgKey(latestPackage), + installSource, + skipPostInstall: true, + }); + if (installResult.error) { + return { name: packageName, error: installResult.error }; + } else { + return { + name: packageName, + version: latestPackage.version, + result: installResult, + }; + } } return { name: packageName, error: result.reason }; }) @@ -56,18 +61,27 @@ export async function bulkInstallPackages({ // only install index patterns if we completed install for any package-version for the // first time, aka fresh installs or upgrades if ( - installResults.find( - (result) => result.status === 'fulfilled' && result.value.result?.status === 'installed' + bulkInstallResults.find( + (result) => + result.status === 'fulfilled' && + !result.value.result?.error && + result.value.result?.status === 'installed' ) ) { await installIndexPatterns({ savedObjectsClient, esClient, installSource }); } - return installResults.map((result, index) => { + return bulkInstallResults.map((result, index) => { const packageName = packagesToInstall[index]; - return result.status === 'fulfilled' - ? result.value - : { name: packageName, error: result.reason }; + if (result.status === 'fulfilled') { + if (result.value && result.value.error) { + return { name: packageName, error: result.value.error }; + } else { + return result.value; + } + } else { + return { name: packageName, error: result.reason }; + } }); } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts index fa2ea9e2209ed..f8c91e55fbbb6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/ensure_installed_default_packages.test.ts @@ -77,7 +77,7 @@ describe('ensureInstalledDefaultPackages', () => { return [ { name: mockInstallation.attributes.name, - result: { assets: [], status: 'installed' }, + result: { assets: [], status: 'installed', installType: 'install' }, version: '', statusCode: 200, }, @@ -95,13 +95,13 @@ describe('ensureInstalledDefaultPackages', () => { return [ { name: 'success one', - result: { assets: [], status: 'installed' }, + result: { assets: [], status: 'installed', installType: 'install' }, version: '', statusCode: 200, }, { name: 'success two', - result: { assets: [], status: 'installed' }, + result: { assets: [], status: 'installed', installType: 'install' }, version: '', statusCode: 200, }, @@ -111,7 +111,7 @@ describe('ensureInstalledDefaultPackages', () => { }, { name: 'success three', - result: { assets: [], status: 'installed' }, + result: { assets: [], status: 'installed', installType: 'install' }, version: '', statusCode: 200, }, @@ -134,7 +134,7 @@ describe('ensureInstalledDefaultPackages', () => { return [ { name: 'undefined package', - result: { assets: [], status: 'installed' }, + result: { assets: [], status: 'installed', installType: 'install' }, version: '', statusCode: 200, }, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 4373251a969bc..31d0732096790 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -201,54 +201,62 @@ async function installPackageFromRegistry({ // TODO: change epm API to /packageName/version so we don't need to do this const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey); - // get the currently installed package - const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); - const installType = getInstallType({ pkgVersion, installedPkg }); - - // get latest package version - const latestPackage = await Registry.fetchFindLatestPackage(pkgName); - - // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update - const installOutOfDateVersionOk = - force || ['reinstall', 'reupdate', 'rollback'].includes(installType); + // if an error happens during getInstallType, report that we don't know + let installType: InstallType = 'unknown'; - // if the requested version is the same as installed version, check if we allow it based on - // current installed package status and force flag, if we don't allow it, - // just return the asset references from the existing installation - if ( - installedPkg?.attributes.version === pkgVersion && - installedPkg?.attributes.install_status === 'installed' - ) { - if (!force) { - logger.debug(`${pkgkey} is already installed, skipping installation`); - return { - assets: [ - ...installedPkg.attributes.installed_es, - ...installedPkg.attributes.installed_kibana, - ], - status: 'already_installed', - }; + try { + // get the currently installed package + const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); + installType = getInstallType({ pkgVersion, installedPkg }); + + // get latest package version + const latestPackage = await Registry.fetchFindLatestPackage(pkgName); + + // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update + const installOutOfDateVersionOk = + force || ['reinstall', 'reupdate', 'rollback'].includes(installType); + + // if the requested version is the same as installed version, check if we allow it based on + // current installed package status and force flag, if we don't allow it, + // just return the asset references from the existing installation + if ( + installedPkg?.attributes.version === pkgVersion && + installedPkg?.attributes.install_status === 'installed' + ) { + if (!force) { + logger.debug(`${pkgkey} is already installed, skipping installation`); + return { + assets: [ + ...installedPkg.attributes.installed_es, + ...installedPkg.attributes.installed_kibana, + ], + status: 'already_installed', + installType, + }; + } } - } - // if the requested version is out-of-date of the latest package version, check if we allow it - // if we don't allow it, return an error - if (semverLt(pkgVersion, latestPackage.version)) { - if (!installOutOfDateVersionOk) { - throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`); + // if the requested version is out-of-date of the latest package version, check if we allow it + // if we don't allow it, return an error + if (semverLt(pkgVersion, latestPackage.version)) { + if (!installOutOfDateVersionOk) { + throw new PackageOutdatedError( + `${pkgkey} is out-of-date and cannot be installed or updated` + ); + } + logger.debug( + `${pkgkey} is out-of-date, installing anyway due to ${ + force ? 'force flag' : `install type ${installType}` + }` + ); } - logger.debug( - `${pkgkey} is out-of-date, installing anyway due to ${ - force ? 'force flag' : `install type ${installType}` - }` - ); - } - // get package info - const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion); + // get package info + const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion); - // try installing the package, if there was an error, call error handler and rethrow - try { + // try installing the package, if there was an error, call error handler and rethrow + // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status + // @ts-ignore return _installPackage({ savedObjectsClient, esClient, @@ -257,19 +265,26 @@ async function installPackageFromRegistry({ packageInfo, installType, installSource: 'registry', - }).then((assets) => { - return { assets, status: 'installed' }; - }); + }) + .then((assets) => { + return { assets, status: 'installed', installType }; + }) + .catch(async (err: Error) => { + await handleInstallPackageFailure({ + savedObjectsClient, + error: err, + pkgName, + pkgVersion, + installedPkg, + esClient, + }); + return { error: err, installType }; + }); } catch (e) { - await handleInstallPackageFailure({ - savedObjectsClient, + return { error: e, - pkgName, - pkgVersion, - installedPkg, - esClient, - }); - throw e; + installType, + }; } } @@ -286,46 +301,57 @@ async function installPackageByUpload({ archiveBuffer, contentType, }: InstallUploadedArchiveParams): Promise { - const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType); - - const installedPkg = await getInstallationObject({ - savedObjectsClient, - pkgName: packageInfo.name, - }); + // if an error happens during getInstallType, report that we don't know + let installType: InstallType = 'unknown'; + try { + const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType); - const installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg }); - if (installType !== 'install') { - throw new PackageOperationNotSupportedError( - `Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.` - ); - } + const installedPkg = await getInstallationObject({ + savedObjectsClient, + pkgName: packageInfo.name, + }); - const installSource = 'upload'; - const paths = await unpackBufferToCache({ - name: packageInfo.name, - version: packageInfo.version, - installSource, - archiveBuffer, - contentType, - }); + installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg }); + if (installType !== 'install') { + throw new PackageOperationNotSupportedError( + `Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.` + ); + } - setPackageInfo({ - name: packageInfo.name, - version: packageInfo.version, - packageInfo, - }); + const installSource = 'upload'; + const paths = await unpackBufferToCache({ + name: packageInfo.name, + version: packageInfo.version, + installSource, + archiveBuffer, + contentType, + }); - return _installPackage({ - savedObjectsClient, - esClient, - installedPkg, - paths, - packageInfo, - installType, - installSource, - }).then((assets) => { - return { assets, status: 'installed' }; - }); + setPackageInfo({ + name: packageInfo.name, + version: packageInfo.version, + packageInfo, + }); + // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status + // @ts-ignore + return _installPackage({ + savedObjectsClient, + esClient, + installedPkg, + paths, + packageInfo, + installType, + installSource, + }) + .then((assets) => { + return { assets, status: 'installed', installType }; + }) + .catch(async (err: Error) => { + return { error: err, installType }; + }); + } catch (e) { + return { error: e, installType }; + } } export type InstallPackageParams = { @@ -352,7 +378,7 @@ export async function installPackage(args: InstallPackageParams) { esClient, force, }).then(async (installResult) => { - if (skipPostInstall) { + if (skipPostInstall || installResult.error) { return installResult; } logger.debug(`install of ${pkgkey} finished, running post-install`); @@ -374,7 +400,7 @@ export async function installPackage(args: InstallPackageParams) { archiveBuffer, contentType, }).then(async (installResult) => { - if (skipPostInstall) { + if (skipPostInstall || installResult.error) { return installResult; } logger.debug(`install of uploaded package finished, running post-install`); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index de798e822b029..706f1bbbaaf35 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -79,6 +79,7 @@ export async function removeInstallation(options: { return installedAssets; } +// TODO: this is very much like deleteKibanaSavedObjectsAssets below function deleteKibanaAssets( installedObjects: KibanaAssetReference[], savedObjectsClient: SavedObjectsClientContract @@ -136,6 +137,7 @@ async function deleteTemplate(esClient: ElasticsearchClient, name: string): Prom } } +// TODO: this is very much like deleteKibanaAssets above export async function deleteKibanaSavedObjectsAssets( savedObjectsClient: SavedObjectsClientContract, installedRefs: AssetReference[] @@ -153,6 +155,9 @@ export async function deleteKibanaSavedObjectsAssets( try { await Promise.all(deletePromises); } catch (err) { - logger.warn(err); + // in the rollback case, partial installs are likely, so missing assets are not an error + if (!savedObjectsClient.errors.isNotFoundError(err)) { + logger.error(err); + } } } diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 009e1a2dad5f1..445d9706bb9a9 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -24,5 +24,6 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./update_assets')); loadTestFile(require.resolve('./data_stream')); loadTestFile(require.resolve('./package_install_complete')); + loadTestFile(require.resolve('./install_error_rollback')); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts new file mode 100644 index 0000000000000..6e2ea3b96aa58 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const goodPackage = 'error_handling-0.1.0'; + const badPackage = 'error_handling-0.2.0'; + + const installPackage = async (pkgkey: string) => { + await supertest + .post(`/api/fleet/epm/packages/${pkgkey}`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }); + }; + + const getPackageInfo = async (pkgkey: string) => { + return await supertest.get(`/api/fleet/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); + }; + + describe('package installation error handling and rollback', async () => { + skipIfNoDockerRegistry(providerContext); + beforeEach(async () => { + await esArchiver.load('empty_kibana'); + }); + afterEach(async () => { + await esArchiver.unload('empty_kibana'); + }); + + it('on a fresh install, it should uninstall a broken package during rollback', async function () { + await supertest + .post(`/api/fleet/epm/packages/${badPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana + + const pkgInfoResponse = await getPackageInfo(badPackage); + expect(JSON.parse(pkgInfoResponse.text).response.status).to.be('not_installed'); + }); + + it('on an upgrade, it should fall back to the previous good version during rollback', async function () { + await installPackage(goodPackage); + await supertest + .post(`/api/fleet/epm/packages/${badPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana + + const goodPkgInfoResponse = await getPackageInfo(goodPackage); + expect(JSON.parse(goodPkgInfoResponse.text).response.status).to.be('installed'); + expect(JSON.parse(goodPkgInfoResponse.text).response.version).to.be('0.1.0'); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/docs/README.md new file mode 100644 index 0000000000000..260499f4b0078 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/docs/README.md @@ -0,0 +1,3 @@ +This package should install without errors. + +Version 0.2.0 of this package should fail during installation. We need this good version to test rollback. \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/kibana/visualization/sample_visualization.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/kibana/visualization/sample_visualization.json new file mode 100644 index 0000000000000..01afe600853ef --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/kibana/visualization/sample_visualization.json @@ -0,0 +1,14 @@ +{ + "attributes": { + "description": "sample visualization", + "title": "sample vis title", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "sample_visualization", + "type": "visualization", + "migrationVersion": { + "visualization": "7.7.0" + } +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml new file mode 100644 index 0000000000000..bba1a6a4c347d --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.1.0/manifest.yml @@ -0,0 +1,20 @@ +format_version: 1.0.0 +name: error_handling +title: Error handling +description: tests error handling and rollback +version: 0.1.0 +categories: [] +release: beta +type: integration +license: basic + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_overrides_64_color.svg' + size: '16x16' + type: 'image/svg+xml' \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/docs/README.md new file mode 100644 index 0000000000000..c348f801b1780 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/docs/README.md @@ -0,0 +1,5 @@ +This package should fail during installation. + +Version 0.1.0 of this package should install without errors, and be rolled back to without errors. + +This package contains one Kibana visualization that requires a non-existent version of Kibana in order to trigger an error during installation. \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/kibana/visualization/sample_visualization.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/kibana/visualization/sample_visualization.json new file mode 100644 index 0000000000000..0a4867cfe1c11 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/kibana/visualization/sample_visualization.json @@ -0,0 +1,14 @@ +{ + "attributes": { + "description": "sample visualization", + "title": "sample vis title", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "sample_visualization", + "type": "visualization", + "migrationVersion": { + "visualization": "12.7.0" + } +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml new file mode 100644 index 0000000000000..2eb6a41a77ede --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.2.0/manifest.yml @@ -0,0 +1,19 @@ +format_version: 1.0.0 +name: error_handling +title: Error handling +description: tests error handling and rollback +version: 0.2.0 +categories: [] +release: beta +type: integration +license: basic + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_overrides_64_color.svg' + size: '16x16' \ No newline at end of file From f8838e3b89abbcf155c9a2381ad631af69cc4864 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Sun, 18 Apr 2021 20:42:07 +0200 Subject: [PATCH 23/25] Remove legacy ES client usages in `home` and `xpack_legacy` (#97359) * Home plugin: remove usages of the legacy ES client * remove legacy es client usage in xpack_legacy --- .../services/sample_data/routes/install.ts | 20 +++++++++---------- .../services/sample_data/routes/list.ts | 20 +++++++++---------- .../services/sample_data/routes/uninstall.ts | 10 ++++------ .../services/sample_data/usage/collector.ts | 17 ++++++---------- .../server/routes/settings.test.ts | 14 ++----------- .../xpack_legacy/server/routes/settings.ts | 2 -- 6 files changed, 30 insertions(+), 53 deletions(-) diff --git a/src/plugins/home/server/services/sample_data/routes/install.ts b/src/plugins/home/server/services/sample_data/routes/install.ts index a20c3e350222f..e5ff33d5c199d 100644 --- a/src/plugins/home/server/services/sample_data/routes/install.ts +++ b/src/plugins/home/server/services/sample_data/routes/install.ts @@ -7,7 +7,7 @@ */ import { schema } from '@kbn/config-schema'; -import { IRouter, Logger, RequestHandlerContext } from 'src/core/server'; +import { IRouter, Logger, IScopedClusterClient } from 'src/core/server'; import { SampleDatasetSchema } from '../lib/sample_dataset_registry_types'; import { createIndexName } from '../lib/create_index_name'; import { @@ -22,7 +22,7 @@ const insertDataIntoIndex = ( dataIndexConfig: any, index: string, nowReference: string, - context: RequestHandlerContext, + esClient: IScopedClusterClient, logger: Logger ) => { function updateTimestamps(doc: any) { @@ -51,9 +51,11 @@ const insertDataIntoIndex = ( bulk.push(insertCmd); bulk.push(updateTimestamps(doc)); }); - const resp = await context.core.elasticsearch.legacy.client.callAsCurrentUser('bulk', { + + const { body: resp } = await esClient.asCurrentUser.bulk({ body: bulk, }); + if (resp.errors) { const errMsg = `sample_data install errors while bulk inserting. Elasticsearch response: ${JSON.stringify( resp, @@ -100,7 +102,7 @@ export function createInstallRoute( // clean up any old installation of dataset try { - await context.core.elasticsearch.legacy.client.callAsCurrentUser('indices.delete', { + await context.core.elasticsearch.client.asCurrentUser.indices.delete({ index, }); } catch (err) { @@ -108,17 +110,13 @@ export function createInstallRoute( } try { - const createIndexParams = { + await context.core.elasticsearch.client.asCurrentUser.indices.create({ index, body: { settings: { index: { number_of_shards: 1, auto_expand_replicas: '0-1' } }, mappings: { properties: dataIndexConfig.fields }, }, - }; - await context.core.elasticsearch.legacy.client.callAsCurrentUser( - 'indices.create', - createIndexParams - ); + }); } catch (err) { const errMsg = `Unable to create sample data index "${index}", error: ${err.message}`; logger.warn(errMsg); @@ -130,7 +128,7 @@ export function createInstallRoute( dataIndexConfig, index, nowReference, - context, + context.core.elasticsearch.client, logger ); (counts as any)[index] = count; diff --git a/src/plugins/home/server/services/sample_data/routes/list.ts b/src/plugins/home/server/services/sample_data/routes/list.ts index 86e286644f936..72d8c31cbafd7 100644 --- a/src/plugins/home/server/services/sample_data/routes/list.ts +++ b/src/plugins/home/server/services/sample_data/routes/list.ts @@ -36,22 +36,20 @@ export const createListRoute = (router: IRouter, sampleDatasets: SampleDatasetSc const dataIndexConfig = sampleDataset.dataIndices[i]; const index = createIndexName(sampleDataset.id, dataIndexConfig.id); try { - const indexExists = await context.core.elasticsearch.legacy.client.callAsCurrentUser( - 'indices.exists', - { index } - ); + const { + body: indexExists, + } = await context.core.elasticsearch.client.asCurrentUser.indices.exists({ + index, + }); if (!indexExists) { sampleDataset.status = NOT_INSTALLED; return; } - const { count } = await context.core.elasticsearch.legacy.client.callAsCurrentUser( - 'count', - { - index, - } - ); - if (count === 0) { + const { body: count } = await context.core.elasticsearch.client.asCurrentUser.count({ + index, + }); + if (count.count === 0) { sampleDataset.status = NOT_INSTALLED; return; } diff --git a/src/plugins/home/server/services/sample_data/routes/uninstall.ts b/src/plugins/home/server/services/sample_data/routes/uninstall.ts index aa8ed67cf840a..3108c06492dd8 100644 --- a/src/plugins/home/server/services/sample_data/routes/uninstall.ts +++ b/src/plugins/home/server/services/sample_data/routes/uninstall.ts @@ -28,11 +28,7 @@ export function createUninstallRoute( async ( { core: { - elasticsearch: { - legacy: { - client: { callAsCurrentUser }, - }, - }, + elasticsearch: { client: esClient }, savedObjects: { getClient: getSavedObjectsClient, typeRegistry }, }, }, @@ -50,7 +46,9 @@ export function createUninstallRoute( const index = createIndexName(sampleDataset.id, dataIndexConfig.id); try { - await callAsCurrentUser('indices.delete', { index }); + await esClient.asCurrentUser.indices.delete({ + index, + }); } catch (err) { return response.customError({ statusCode: err.status, diff --git a/src/plugins/home/server/services/sample_data/usage/collector.ts b/src/plugins/home/server/services/sample_data/usage/collector.ts index 81958a2e3c878..df7d485c1f6fa 100644 --- a/src/plugins/home/server/services/sample_data/usage/collector.ts +++ b/src/plugins/home/server/services/sample_data/usage/collector.ts @@ -6,22 +6,17 @@ * Side Public License, v 1. */ -import { PluginInitializerContext } from 'kibana/server'; -import { first } from 'rxjs/operators'; +import type { PluginInitializerContext } from 'kibana/server'; +import type { UsageCollectionSetup } from '../../../../../usage_collection/server'; import { fetchProvider, TelemetryResponse } from './collector_fetch'; -import { UsageCollectionSetup } from '../../../../../usage_collection/server'; -export async function makeSampleDataUsageCollector( +export function makeSampleDataUsageCollector( usageCollection: UsageCollectionSetup, context: PluginInitializerContext ) { - let index: string; - try { - const config = await context.config.legacy.globalConfig$.pipe(first()).toPromise(); - index = config.kibana.index; - } catch (err) { - return; // kibana plugin is not enabled (test environment) - } + const config = context.config.legacy.get(); + const index = config.kibana.index; + const collector = usageCollection.makeUsageCollector({ type: 'sample-data', fetch: fetchProvider(index), diff --git a/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts b/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts index 08b5a0f60521c..2034a4e5b74ba 100644 --- a/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts +++ b/x-pack/plugins/xpack_legacy/server/routes/settings.test.ts @@ -9,11 +9,7 @@ import { BehaviorSubject } from 'rxjs'; import { UnwrapPromise } from '@kbn/utility-types'; import supertest from 'supertest'; -import { - LegacyAPICaller, - ServiceStatus, - ServiceStatusLevels, -} from '../../../../../src/core/server'; +import { ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; import { contextServiceMock, elasticsearchServiceMock, @@ -31,24 +27,18 @@ export function mockGetClusterInfo(clusterInfo: any) { esClient.info.mockResolvedValue({ body: { ...clusterInfo } }); return esClient; } + describe('/api/settings', () => { let server: HttpService; let httpSetup: HttpSetup; let overallStatus$: BehaviorSubject; - let mockApiCaller: jest.Mocked; beforeEach(async () => { - mockApiCaller = jest.fn(); server = createHttpServer(); httpSetup = await server.setup({ context: contextServiceMock.createSetupContract({ core: { elasticsearch: { - legacy: { - client: { - callAsCurrentUser: mockApiCaller, - }, - }, client: { asCurrentUser: mockGetClusterInfo({ cluster_uuid: 'yyy-yyyyy' }), }, diff --git a/x-pack/plugins/xpack_legacy/server/routes/settings.ts b/x-pack/plugins/xpack_legacy/server/routes/settings.ts index 9117637b70bee..b9052ca0c84e3 100644 --- a/x-pack/plugins/xpack_legacy/server/routes/settings.ts +++ b/x-pack/plugins/xpack_legacy/server/routes/settings.ts @@ -42,9 +42,7 @@ export function registerSettingsRoute({ validate: false, }, async (context, req, res) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; const collectorFetchContext = { - callCluster: callAsCurrentUser, esClient: context.core.elasticsearch.client.asCurrentUser, soClient: context.core.savedObjects.client, }; From cb2cf67609f54a6e43a08b24f11694217255cdc3 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Sun, 18 Apr 2021 20:49:35 +0200 Subject: [PATCH 24/25] Add description as title on tag badge (#97109) --- .../public/components/base/tag_badge.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/saved_objects_tagging/public/components/base/tag_badge.tsx b/x-pack/plugins/saved_objects_tagging/public/components/base/tag_badge.tsx index 6bc9e659d9346..e8af661d6921d 100644 --- a/x-pack/plugins/saved_objects_tagging/public/components/base/tag_badge.tsx +++ b/x-pack/plugins/saved_objects_tagging/public/components/base/tag_badge.tsx @@ -17,5 +17,9 @@ export interface TagBadgeProps { * The badge representation of a Tag, which is the default display to be used for them. */ export const TagBadge: FC = ({ tag }) => { - return {tag.name}; + return ( + + {tag.name} + + ); }; From 787b4934032b6989195aedf3aac4c871bf7ca11f Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Sun, 18 Apr 2021 23:07:36 +0200 Subject: [PATCH 25/25] Avoid mutating KQL query when validating it (#97081) --- .../service/lib/filter_utils.test.ts | 17 +++++++++++++++++ .../saved_objects/service/lib/filter_utils.ts | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/filter_utils.test.ts b/src/core/server/saved_objects/service/lib/filter_utils.test.ts index 956a60b23809d..2ef5219ccfff1 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.test.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.test.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { cloneDeep } from 'lodash'; // @ts-expect-error no ts import { esKuery } from '../../es_query'; @@ -105,6 +106,22 @@ describe('Filter Utils', () => { ) ).toEqual(esKuery.fromKueryExpression('foo.title: "best"')); }); + + test('does not mutate the input KueryNode', () => { + const input = esKuery.nodeTypes.function.buildNode( + 'is', + `foo.attributes.title`, + 'best', + true + ); + + const inputCopy = cloneDeep(input); + + validateConvertFilterToKueryNode(['foo'], input, mockMappings); + + expect(input).toEqual(inputCopy); + }); + test('Validate a simple KQL expression filter', () => { expect( validateConvertFilterToKueryNode(['foo'], 'foo.attributes.title: "best"', mockMappings) diff --git a/src/core/server/saved_objects/service/lib/filter_utils.ts b/src/core/server/saved_objects/service/lib/filter_utils.ts index b3bcef9a62e13..a41a25a27b70d 100644 --- a/src/core/server/saved_objects/service/lib/filter_utils.ts +++ b/src/core/server/saved_objects/service/lib/filter_utils.ts @@ -7,11 +7,12 @@ */ import { set } from '@elastic/safer-lodash-set'; -import { get } from 'lodash'; +import { get, cloneDeep } from 'lodash'; import { SavedObjectsErrorHelpers } from './errors'; import { IndexMapping } from '../../mappings'; // @ts-expect-error no ts import { esKuery } from '../../es_query'; + type KueryNode = any; const astFunctionType = ['is', 'range', 'nested']; @@ -23,7 +24,7 @@ export const validateConvertFilterToKueryNode = ( ): KueryNode | undefined => { if (filter && indexMapping) { const filterKueryNode = - typeof filter === 'string' ? esKuery.fromKueryExpression(filter) : filter; + typeof filter === 'string' ? esKuery.fromKueryExpression(filter) : cloneDeep(filter); const validationFilterKuery = validateFilterKueryNode({ astFilter: filterKueryNode,