diff --git a/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts b/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts index 1c54e543e7747..f3b332a5930fc 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts @@ -29,6 +29,7 @@ import { } from './mocks'; import { bulkCreateArtifacts, + bulkDeleteArtifacts, createArtifact, deleteArtifact, encodeArtifactContent, @@ -348,6 +349,54 @@ describe('When using the artifacts services', () => { }); }); + describe('and calling `bulkDeleteArtifacts()`', () => { + it('should delete single artifact', async () => { + bulkDeleteArtifacts(esClientMock, ['123']); + + expect(esClientMock.bulk).toHaveBeenCalledWith({ + refresh: 'wait_for', + body: [ + { + delete: { + _id: '123', + _index: FLEET_SERVER_ARTIFACTS_INDEX, + }, + }, + ], + }); + }); + + it('should delete all the artifacts', async () => { + bulkDeleteArtifacts(esClientMock, ['123', '231']); + + expect(esClientMock.bulk).toHaveBeenCalledWith({ + refresh: 'wait_for', + body: [ + { + delete: { + _id: '123', + _index: FLEET_SERVER_ARTIFACTS_INDEX, + }, + }, + { + delete: { + _id: '231', + _index: FLEET_SERVER_ARTIFACTS_INDEX, + }, + }, + ], + }); + }); + + it('should throw an ArtifactElasticsearchError if one is encountered', async () => { + setEsClientMethodResponseToError(esClientMock, 'bulk'); + + await expect(bulkDeleteArtifacts(esClientMock, ['123'])).rejects.toBeInstanceOf( + ArtifactsElasticsearchError + ); + }); + }); + describe('and calling `listArtifacts()`', () => { beforeEach(() => { esClientMock.search.mockResponse(generateArtifactEsSearchResultHitsMock()); diff --git a/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts b/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts index 0e312f142edf6..5516ab6f70e23 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts @@ -197,6 +197,37 @@ export const deleteArtifact = async (esClient: ElasticsearchClient, id: string): } }; +export const bulkDeleteArtifacts = async ( + esClient: ElasticsearchClient, + ids: string[] +): Promise => { + try { + const body = ids.map((id) => ({ + delete: { _index: FLEET_SERVER_ARTIFACTS_INDEX, _id: id }, + })); + + const res = await withPackageSpan(`Bulk delete fleet artifacts`, () => + esClient.bulk({ + body, + refresh: 'wait_for', + }) + ); + let errors: Error[] = []; + // Track errors of the bulk delete action + if (res.errors) { + errors = res.items.reduce((acc, item) => { + if (item.delete?.error) { + acc.push(new Error(item.delete.error.reason)); + } + return acc; + }, []); + } + return errors; + } catch (e) { + throw new ArtifactsElasticsearchError(e); + } +}; + export const listArtifacts = async ( esClient: ElasticsearchClient, options: ListArtifactsProps = {} diff --git a/x-pack/plugins/fleet/server/services/artifacts/client.test.ts b/x-pack/plugins/fleet/server/services/artifacts/client.test.ts index 1a242c83275fd..61d07f0a3e99c 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/client.test.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/client.test.ts @@ -7,6 +7,8 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { FLEET_SERVER_ARTIFACTS_INDEX } from '../../../common/constants'; + import { ArtifactsClientAccessDeniedError, ArtifactsClientError } from '../../errors'; import { appContextService } from '../app_context'; @@ -155,6 +157,25 @@ describe('When using the Fleet Artifacts Client', () => { }); }); + describe('and calling `bulkDeleteArtifacts()`', () => { + it('should bulk delete the artifact', async () => { + setEsClientGetMock(); + await artifactClient.bulkDeleteArtifacts(['123']); + expect(esClientMock.bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: [ + { + delete: { + _id: 'endpoint:123', + _index: FLEET_SERVER_ARTIFACTS_INDEX, + }, + }, + ], + }) + ); + }); + }); + describe('and calling `listArtifacts()`', () => { beforeEach(() => { esClientMock.search.mockResponse(generateArtifactEsSearchResultHitsMock()); diff --git a/x-pack/plugins/fleet/server/services/artifacts/client.ts b/x-pack/plugins/fleet/server/services/artifacts/client.ts index 3cf28e4847d0d..7ba2452e83fe7 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/client.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/client.ts @@ -18,7 +18,7 @@ import type { NewArtifact, ListArtifactsProps, } from './types'; -import { relativeDownloadUrlFromArtifact } from './mappings'; +import { relativeDownloadUrlFromArtifact, uniqueIdFromId } from './mappings'; import { createArtifact, @@ -28,6 +28,7 @@ import { getArtifact, listArtifacts, bulkCreateArtifacts, + bulkDeleteArtifacts, } from './artifacts'; /** @@ -112,6 +113,11 @@ export class FleetArtifactsClient implements ArtifactsClientInterface { } } + async bulkDeleteArtifacts(ids: string[]): Promise { + const idsMappedWithPackageName = ids.map((id) => uniqueIdFromId(id, this.packageName)); + return await bulkDeleteArtifacts(this.esClient, idsMappedWithPackageName); + } + /** * Get a list of artifacts. * NOTE that when using the `kuery` filtering param, that all filters property names should diff --git a/x-pack/plugins/fleet/server/services/artifacts/mappings.ts b/x-pack/plugins/fleet/server/services/artifacts/mappings.ts index 09b804a208342..3645f957417e3 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/mappings.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/mappings.ts @@ -82,3 +82,7 @@ export const uniqueIdFromArtifact = < }: T): string => { return `${packageName}:${identifier}-${decodedSha256}`; }; + +export const uniqueIdFromId = (id: string, packageName: string): string => { + return `${packageName}:${id}`; +}; diff --git a/x-pack/plugins/fleet/server/services/artifacts/mocks.ts b/x-pack/plugins/fleet/server/services/artifacts/mocks.ts index 26dc93181a102..dc831558cb7bb 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/mocks.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/mocks.ts @@ -26,6 +26,7 @@ export const createArtifactsClientMock = (): jest.Mocked>( }; type EsClientMock = ReturnType; -type EsClientMockMethods = keyof Pick; +type EsClientMockMethods = keyof Pick< + EsClientMock, + 'get' | 'create' | 'delete' | 'search' | 'bulk' +>; export const setEsClientMethodResponseToError = ( esClientMock: EsClientMock, diff --git a/x-pack/plugins/fleet/server/services/artifacts/types.ts b/x-pack/plugins/fleet/server/services/artifacts/types.ts index 05ba89b3bb0ff..4b0aacd92bc20 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/types.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/types.ts @@ -86,6 +86,8 @@ export interface ArtifactsClientInterface { deleteArtifact(id: string): Promise; + bulkDeleteArtifacts(ids: string[]): Promise; + listArtifacts(options?: ListArtifactsProps): Promise>; encodeContent(content: ArtifactsClientCreateOptions['content']): Promise; diff --git a/x-pack/plugins/security_solution/server/config.mock.ts b/x-pack/plugins/security_solution/server/config.mock.ts index 88ce4c7b91910..6b2d4b5bff3b3 100644 --- a/x-pack/plugins/security_solution/server/config.mock.ts +++ b/x-pack/plugins/security_solution/server/config.mock.ts @@ -21,6 +21,7 @@ export const createMockConfig = (): ConfigType => { maxTimelineImportPayloadBytes: 10485760, enableExperimental, packagerTaskInterval: '60s', + packagerTaskPackagePolicyUpdateBatchSize: 10, prebuiltRulesPackageVersion: '', alertMergeStrategy: 'missingFields', alertIgnoreFields: [], diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index dc732061ab947..ce177538f4f0b 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -95,6 +95,11 @@ export const configSchema = schema.object({ */ packagerTaskInterval: schema.string({ defaultValue: '60s' }), + /** + * Artifacts Configuration for package policy update concurrency + */ + packagerTaskPackagePolicyUpdateBatchSize: schema.number({ defaultValue: 10, max: 50, min: 1 }), + /** * For internal use. Specify which version of the Detection Rules fleet package to install * when upgrading rules. If not provided, the latest compatible package will be installed, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 3ef1522109d35..bb46bc2da92b4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -10,7 +10,12 @@ import { listMock } from '@kbn/lists-plugin/server/mocks'; import { getFoundExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/found_exception_list_item_schema.mock'; import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock'; import type { EntriesArray, EntryList } from '@kbn/securitysolution-io-ts-list-types'; -import { buildArtifact, getEndpointExceptionList, getFilteredEndpointExceptionList } from './lists'; +import { + buildArtifact, + getAllItemsFromEndpointExceptionList, + getFilteredEndpointExceptionListRaw, + convertExceptionsToEndpointFormat, +} from './lists'; import type { TranslatedEntry, TranslatedExceptionListItem } from '../../schemas/artifacts'; import { ArtifactConstants } from './common'; import { @@ -29,10 +34,10 @@ describe('artifacts lists', () => { mockExceptionClient = listMock.getExceptionListClient(); }); - describe('getFilteredEndpointExceptionList', () => { + describe('getFilteredEndpointExceptionListRaw', () => { const TEST_FILTER = 'exception-list-agnostic.attributes.os_types:"linux"'; - test('it should convert the exception lists response to the proper endpoint format', async () => { + test('it should get convert the exception lists response to the proper endpoint format', async () => { const expectedEndpointExceptions = { type: 'simple', entries: [ @@ -59,13 +64,13 @@ describe('artifacts lists', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -105,13 +110,13 @@ describe('artifacts lists', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -156,13 +161,13 @@ describe('artifacts lists', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -209,13 +214,13 @@ describe('artifacts lists', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -261,13 +266,13 @@ describe('artifacts lists', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -304,13 +309,13 @@ describe('artifacts lists', () => { first.data[1].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -347,13 +352,13 @@ describe('artifacts lists', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -376,15 +381,15 @@ describe('artifacts lists', () => { .mockReturnValueOnce(first) .mockReturnValueOnce(second); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); - // Expect 2 exceptions, the first two calls returned the same exception list items - expect(resp.entries.length).toEqual(2); + // Expect 1 exceptions, the first two calls returned the same exception list items + expect(translated.entries.length).toEqual(1); }); test('it should handle no exceptions', async () => { @@ -392,13 +397,13 @@ describe('artifacts lists', () => { exceptionsResponse.data = []; exceptionsResponse.total = 0; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: TEST_FILTER, listId: ENDPOINT_LIST_ID, }); - expect(resp.entries.length).toEqual(0); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated.entries.length).toEqual(0); }); test('it should return a stable hash regardless of order of entries', async () => { @@ -552,13 +557,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -597,13 +602,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -636,13 +641,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -700,14 +705,14 @@ describe('artifacts lists', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); - expect(resp).toEqual({ + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -740,13 +745,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -804,14 +809,14 @@ describe('artifacts lists', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -854,13 +859,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -901,13 +906,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -942,13 +947,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -1008,14 +1013,14 @@ describe('artifacts lists', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -1049,13 +1054,13 @@ describe('artifacts lists', () => { first.data[0].os_types = [os]; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -1115,14 +1120,14 @@ describe('artifacts lists', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFilteredEndpointExceptionList({ + const resp = await getFilteredEndpointExceptionListRaw({ elClient: mockExceptionClient, - schemaVersion: 'v1', filter: `${getOsFilter(os)} and (exception-list-agnostic.attributes.tags:"policy:all")`, listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); - expect(resp).toEqual({ + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual({ entries: [expectedEndpointExceptions], }); }); @@ -1157,25 +1162,26 @@ describe('artifacts lists', () => { ], }; - describe('Builds proper kuery without policy', () => { + describe('Builds proper kuery', () => { test('for Endpoint List', async () => { mockExceptionClient.findExceptionListItem = jest .fn() .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - const resp = await getEndpointExceptionList({ + const resp = await getAllItemsFromEndpointExceptionList({ elClient: mockExceptionClient, - schemaVersion: 'v1', os: 'windows', + listId: ENDPOINT_LIST_ID, }); - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual(TEST_EXCEPTION_LIST_ITEM); expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ listId: ENDPOINT_LIST_ID, namespaceType: 'agnostic', filter: 'exception-list-agnostic.attributes.os_types:"windows"', - perPage: 100, + perPage: 1000, page: 1, sortField: 'created_at', sortOrder: 'desc', @@ -1187,21 +1193,20 @@ describe('artifacts lists', () => { .fn() .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - const resp = await getEndpointExceptionList({ + const resp = await getAllItemsFromEndpointExceptionList({ elClient: mockExceptionClient, - schemaVersion: 'v1', os: 'macos', listId: ENDPOINT_TRUSTED_APPS_LIST_ID, }); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); + expect(translated).toEqual(TEST_EXCEPTION_LIST_ITEM); expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ listId: ENDPOINT_TRUSTED_APPS_LIST_ID, namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and (exception-list-agnostic.attributes.tags:"policy:all")', - perPage: 100, + filter: 'exception-list-agnostic.attributes.os_types:"macos"', + perPage: 1000, page: 1, sortField: 'created_at', sortOrder: 'desc', @@ -1213,21 +1218,20 @@ describe('artifacts lists', () => { .fn() .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - const resp = await getEndpointExceptionList({ + const resp = await getAllItemsFromEndpointExceptionList({ elClient: mockExceptionClient, - schemaVersion: 'v1', os: 'macos', listId: ENDPOINT_EVENT_FILTERS_LIST_ID, }); - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual(TEST_EXCEPTION_LIST_ITEM); expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ listId: ENDPOINT_EVENT_FILTERS_LIST_ID, namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and (exception-list-agnostic.attributes.tags:"policy:all")', - perPage: 100, + filter: 'exception-list-agnostic.attributes.os_types:"macos"', + perPage: 1000, page: 1, sortField: 'created_at', sortOrder: 'desc', @@ -1239,164 +1243,45 @@ describe('artifacts lists', () => { .fn() .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - const resp = await getEndpointExceptionList({ + const resp = await getAllItemsFromEndpointExceptionList({ elClient: mockExceptionClient, - schemaVersion: 'v1', os: 'macos', listId: ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, }); - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual(TEST_EXCEPTION_LIST_ITEM); expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ listId: ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and (exception-list-agnostic.attributes.tags:"policy:all")', - perPage: 100, - page: 1, - sortField: 'created_at', - sortOrder: 'desc', - }); - }); - - test('for Blocklists', async () => { - mockExceptionClient.findExceptionListItem = jest - .fn() - .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - - const resp = await getEndpointExceptionList({ - elClient: mockExceptionClient, - schemaVersion: 'v1', - os: 'macos', - listId: ENDPOINT_BLOCKLISTS_LIST_ID, - }); - - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); - - expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ - listId: ENDPOINT_BLOCKLISTS_LIST_ID, - namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and (exception-list-agnostic.attributes.tags:"policy:all")', - perPage: 100, - page: 1, - sortField: 'created_at', - sortOrder: 'desc', - }); - }); - }); - - describe('Build proper kuery with policy', () => { - test('for Trusted Apps', async () => { - mockExceptionClient.findExceptionListItem = jest - .fn() - .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - - const resp = await getEndpointExceptionList({ - elClient: mockExceptionClient, - schemaVersion: 'v1', - os: 'macos', - policyId: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', - listId: ENDPOINT_TRUSTED_APPS_LIST_ID, - }); - - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); - - expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ - listId: ENDPOINT_TRUSTED_APPS_LIST_ID, - namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and ' + - '(exception-list-agnostic.attributes.tags:"policy:all" or ' + - 'exception-list-agnostic.attributes.tags:"policy:c6d16e42-c32d-4dce-8a88-113cfe276ad1")', - perPage: 100, + filter: 'exception-list-agnostic.attributes.os_types:"macos"', + perPage: 1000, page: 1, sortField: 'created_at', sortOrder: 'desc', }); }); - test('for Event Filters', async () => { - mockExceptionClient.findExceptionListItem = jest - .fn() - .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - - const resp = await getEndpointExceptionList({ - elClient: mockExceptionClient, - schemaVersion: 'v1', - os: 'macos', - policyId: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', - listId: ENDPOINT_EVENT_FILTERS_LIST_ID, - }); - - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); - - expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ - listId: ENDPOINT_EVENT_FILTERS_LIST_ID, - namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and ' + - '(exception-list-agnostic.attributes.tags:"policy:all" or ' + - 'exception-list-agnostic.attributes.tags:"policy:c6d16e42-c32d-4dce-8a88-113cfe276ad1")', - perPage: 100, - page: 1, - sortField: 'created_at', - sortOrder: 'desc', - }); - }); - - test('for Host Isolation Exceptions', async () => { - mockExceptionClient.findExceptionListItem = jest - .fn() - .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - - const resp = await getEndpointExceptionList({ - elClient: mockExceptionClient, - schemaVersion: 'v1', - os: 'macos', - policyId: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', - listId: ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, - }); - - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); - - expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ - listId: ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, - namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and ' + - '(exception-list-agnostic.attributes.tags:"policy:all" or ' + - 'exception-list-agnostic.attributes.tags:"policy:c6d16e42-c32d-4dce-8a88-113cfe276ad1")', - perPage: 100, - page: 1, - sortField: 'created_at', - sortOrder: 'desc', - }); - }); test('for Blocklists', async () => { mockExceptionClient.findExceptionListItem = jest .fn() .mockReturnValueOnce(getFoundExceptionListItemSchemaMock()); - const resp = await getEndpointExceptionList({ + const resp = await getAllItemsFromEndpointExceptionList({ elClient: mockExceptionClient, - schemaVersion: 'v1', os: 'macos', - policyId: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', listId: ENDPOINT_BLOCKLISTS_LIST_ID, }); - expect(resp).toEqual(TEST_EXCEPTION_LIST_ITEM); + const translated = convertExceptionsToEndpointFormat(resp, 'v1'); + expect(translated).toEqual(TEST_EXCEPTION_LIST_ITEM); expect(mockExceptionClient.findExceptionListItem).toHaveBeenCalledWith({ listId: ENDPOINT_BLOCKLISTS_LIST_ID, namespaceType: 'agnostic', - filter: - 'exception-list-agnostic.attributes.os_types:"macos" and ' + - '(exception-list-agnostic.attributes.tags:"policy:all" or ' + - 'exception-list-agnostic.attributes.tags:"policy:c6d16e42-c32d-4dce-8a88-113cfe276ad1")', - perPage: 100, + filter: 'exception-list-agnostic.attributes.os_types:"macos"', + perPage: 1000, page: 1, sortField: 'created_at', sortOrder: 'desc', diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 581b46deebcfd..f8fe9d6e71453 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -11,8 +11,8 @@ import type { Entry, EntryNested, ExceptionListItemSchema, + FoundExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { validate } from '@kbn/securitysolution-io-ts-utils'; import type { OperatingSystem } from '@kbn/securitysolution-utils'; import { hasSimpleExecutableName } from '@kbn/securitysolution-utils'; @@ -20,10 +20,11 @@ import type { ENDPOINT_BLOCKLISTS_LIST_ID, ENDPOINT_EVENT_FILTERS_LIST_ID, ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, + ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID, } from '@kbn/securitysolution-list-constants'; -import { ENDPOINT_LIST_ID } from '@kbn/securitysolution-list-constants'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; +import { validate } from '@kbn/securitysolution-io-ts-utils'; import type { InternalArtifactCompleteSchema, TranslatedEntry, @@ -74,18 +75,31 @@ export type ArtifactListId = | typeof ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID | typeof ENDPOINT_BLOCKLISTS_LIST_ID; -export async function getFilteredEndpointExceptionList({ +export function convertExceptionsToEndpointFormat( + exceptions: ExceptionListItemSchema[], + schemaVersion: string +) { + const translatedExceptions = { + entries: translateToEndpointExceptions(exceptions, schemaVersion), + }; + const [validated, errors] = validate(translatedExceptions, wrappedTranslatedExceptionList); + if (errors != null) { + throw new Error(errors); + } + + return validated as WrappedTranslatedExceptionList; +} + +export async function getFilteredEndpointExceptionListRaw({ elClient, filter, listId, - schemaVersion, }: { elClient: ExceptionListClient; filter: string; listId: ArtifactListId; - schemaVersion: string; -}): Promise { - const exceptions: WrappedTranslatedExceptionList = { entries: [] }; +}): Promise { + let exceptions: ExceptionListItemSchema[] = []; let page = 1; let paging = true; @@ -94,63 +108,39 @@ export async function getFilteredEndpointExceptionList({ listId, namespaceType: 'agnostic', filter, - perPage: 100, + perPage: 1000, page, sortField: 'created_at', sortOrder: 'desc', }); if (response?.data !== undefined) { - exceptions.entries = exceptions.entries.concat( - translateToEndpointExceptions(response.data, schemaVersion) - ); + exceptions = exceptions.concat(response.data); - paging = (page - 1) * 100 + response.data.length < response.total; + paging = (page - 1) * 1000 + response.data.length < response.total; page++; } else { break; } } - const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList); - if (errors != null) { - throw new Error(errors); - } - return validated as WrappedTranslatedExceptionList; + return exceptions; } -export async function getEndpointExceptionList({ +export async function getAllItemsFromEndpointExceptionList({ elClient, listId, os, - policyId, - schemaVersion, }: { elClient: ExceptionListClient; - listId?: ArtifactListId; + listId: ArtifactListId; os: string; - policyId?: string; - schemaVersion: string; -}): Promise { +}): Promise { const osFilter = `exception-list-agnostic.attributes.os_types:\"${os}\"`; - const policyFilter = `(exception-list-agnostic.attributes.tags:\"policy:all\"${ - policyId ? ` or exception-list-agnostic.attributes.tags:\"policy:${policyId}\"` : '' - })`; - // for endpoint list - if (!listId || listId === ENDPOINT_LIST_ID) { - return getFilteredEndpointExceptionList({ - elClient, - schemaVersion, - filter: `${osFilter}`, - listId: ENDPOINT_LIST_ID, - }); - } - // for TAs, EFs, Host IEs and Blocklists - return getFilteredEndpointExceptionList({ + return getFilteredEndpointExceptionListRaw({ elClient, - schemaVersion, - filter: `${osFilter} and ${policyFilter}`, + filter: osFilter, listId, }); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 5df2d67eeb582..ce9e8e2fc46c8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -70,5 +70,17 @@ describe('artifact_client', () => { }); expect(fleetArtifactClient.deleteArtifact).toHaveBeenCalledWith('123'); }); + + test('can bulk delete artifacts', async () => { + await artifactClient.bulkDeleteArtifacts([ + 'endpoint-trustlist-linux-v1-sha26hash', + 'endpoint-trustlist-windows-v1-sha26hash', + ]); + expect(fleetArtifactClient.listArtifacts).toHaveBeenCalledTimes(0); + expect(fleetArtifactClient.bulkDeleteArtifacts).toHaveBeenCalledWith([ + 'endpoint-trustlist-linux-v1-sha26hash', + 'endpoint-trustlist-windows-v1-sha26hash', + ]); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index ed5723ddccf0e..3e00310a5bb64 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -24,6 +24,8 @@ export interface EndpointArtifactClientInterface { deleteArtifact(id: string): Promise; + bulkDeleteArtifacts(ids: string[]): Promise; + listArtifacts(options?: ListArtifactsProps): Promise>; } @@ -96,4 +98,8 @@ export class EndpointArtifactClient implements EndpointArtifactClientInterface { const artifactId = (await this.getArtifact(id))?.id!; return this.fleetArtifacts.deleteArtifact(artifactId); } + + async bulkDeleteArtifacts(ids: string[]): Promise { + return this.fleetArtifacts.bulkDeleteArtifacts(ids); + } } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index aaf16057a0df9..cb947b5221dbb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { savedObjectsClientMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { + savedObjectsClientMock, + loggingSystemMock, + elasticsearchServiceMock, +} from '@kbn/core/server/mocks'; import type { Logger } from '@kbn/core/server'; import type { PackagePolicyClient } from '@kbn/fleet-plugin/server'; import { createPackagePolicyServiceMock } from '@kbn/fleet-plugin/server/mocks'; @@ -88,6 +92,8 @@ export const buildManifestManagerContextMock = ( artifactClient: createEndpointArtifactClientMock(), logger: loggingSystemMock.create().get() as jest.Mocked, experimentalFeatures: parseExperimentalConfigValue([]).features, + packagerTaskPackagePolicyUpdateBatchSize: 10, + esClient: elasticsearchServiceMock.createElasticsearchClient(), }; }; @@ -127,7 +133,7 @@ export const getManifestManagerMock = ( context.exceptionListClient.findExceptionListItem = jest .fn() .mockRejectedValue(new Error('unexpected thing happened')); - return super.buildExceptionListArtifacts(); + return super.buildExceptionListArtifacts([]); case ManifestManagerMockType.NormalFlow: return getMockArtifactsWithDiff(); } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 28933fb35e81c..f6859015f42fc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -349,10 +349,22 @@ describe('ManifestManager', () => { test('Builds fully new manifest if no baseline parameter passed and present exception list items', async () => { const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); - const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); - const eventFiltersListItem = getExceptionListItemSchemaMock({ os_types: ['windows'] }); - const hostIsolationExceptionsItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); - const blocklistsListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); + const trustedAppListItem = getExceptionListItemSchemaMock({ + os_types: ['linux'], + tags: ['policy:all'], + }); + const eventFiltersListItem = getExceptionListItemSchemaMock({ + os_types: ['windows'], + tags: ['policy:all'], + }); + const hostIsolationExceptionsItem = getExceptionListItemSchemaMock({ + os_types: ['linux'], + tags: ['policy:all'], + }); + const blocklistsListItem = getExceptionListItemSchemaMock({ + os_types: ['macos'], + tags: ['policy:all'], + }); const context = buildManifestManagerContextMock({}); const manifestManager = new ManifestManager(context); @@ -417,10 +429,22 @@ describe('ManifestManager', () => { test('Reuses artifacts when baseline parameter passed and present exception list items', async () => { const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); - const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); - const eventFiltersListItem = getExceptionListItemSchemaMock({ os_types: ['windows'] }); - const hostIsolationExceptionsItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); - const blocklistsListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); + const trustedAppListItem = getExceptionListItemSchemaMock({ + os_types: ['linux'], + tags: ['policy:all'], + }); + const eventFiltersListItem = getExceptionListItemSchemaMock({ + os_types: ['windows'], + tags: ['policy:all'], + }); + const hostIsolationExceptionsItem = getExceptionListItemSchemaMock({ + os_types: ['linux'], + tags: ['policy:all'], + }); + const blocklistsListItem = getExceptionListItemSchemaMock({ + os_types: ['macos'], + tags: ['policy:all'], + }); const context = buildManifestManagerContextMock({}); const manifestManager = new ManifestManager(context); @@ -486,15 +510,18 @@ describe('ManifestManager', () => { } }); - // test('Builds manifest with policy specific exception list items for trusted apps', async () => { const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); - const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); + const trustedAppListItem = getExceptionListItemSchemaMock({ + os_types: ['linux'], + tags: ['policy:all'], + }); const trustedAppListItemPolicy2 = getExceptionListItemSchemaMock({ os_types: ['linux'], entries: [ { field: 'other.field', operator: 'included', type: 'match', value: 'other value' }, ], + tags: [`policy:${TEST_POLICY_ID_2}`], }); const context = buildManifestManagerContextMock({}); const manifestManager = new ManifestManager(context); @@ -502,8 +529,7 @@ describe('ManifestManager', () => { context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({ [ENDPOINT_LIST_ID]: { macos: [exceptionListItem] }, [ENDPOINT_TRUSTED_APPS_LIST_ID]: { - linux: [trustedAppListItem], - [`linux-${TEST_POLICY_ID_2}`]: [trustedAppListItem, trustedAppListItemPolicy2], + linux: [trustedAppListItem, trustedAppListItemPolicy2], }, }); context.packagePolicyService.listIds = mockPolicyListIdsResponse([ @@ -574,14 +600,10 @@ describe('ManifestManager', () => { ]) ).resolves.toStrictEqual([]); - expect(context.artifactClient.deleteArtifact).toHaveBeenNthCalledWith( - 1, - ARTIFACT_ID_EXCEPTIONS_MACOS - ); - expect(context.artifactClient.deleteArtifact).toHaveBeenNthCalledWith( - 2, - ARTIFACT_ID_EXCEPTIONS_WINDOWS - ); + expect(context.artifactClient.bulkDeleteArtifacts).toHaveBeenCalledWith([ + ARTIFACT_ID_EXCEPTIONS_MACOS, + ARTIFACT_ID_EXCEPTIONS_WINDOWS, + ]); }); test('Returns errors for partial failures', async () => { @@ -590,10 +612,11 @@ describe('ManifestManager', () => { const manifestManager = new ManifestManager(context); const error = new Error(); - artifactClient.deleteArtifact.mockImplementation(async (id) => { - if (id === ARTIFACT_ID_EXCEPTIONS_WINDOWS) { - throw error; + artifactClient.bulkDeleteArtifacts.mockImplementation(async (ids): Promise => { + if (ids[1] === ARTIFACT_ID_EXCEPTIONS_WINDOWS) { + return [error]; } + return []; }); await expect( @@ -603,15 +626,11 @@ describe('ManifestManager', () => { ]) ).resolves.toStrictEqual([error]); - expect(artifactClient.deleteArtifact).toHaveBeenCalledTimes(2); - expect(artifactClient.deleteArtifact).toHaveBeenNthCalledWith( - 1, - ARTIFACT_ID_EXCEPTIONS_MACOS - ); - expect(artifactClient.deleteArtifact).toHaveBeenNthCalledWith( - 2, - ARTIFACT_ID_EXCEPTIONS_WINDOWS - ); + expect(artifactClient.bulkDeleteArtifacts).toHaveBeenCalledTimes(1); + expect(context.artifactClient.bulkDeleteArtifacts).toHaveBeenCalledWith([ + ARTIFACT_ID_EXCEPTIONS_MACOS, + ARTIFACT_ID_EXCEPTIONS_WINDOWS, + ]); }); }); @@ -790,12 +809,6 @@ describe('ManifestManager', () => { total: items.length, }); - const toNewPackagePolicy = (packagePolicy: PackagePolicy) => { - const { id, revision, updated_at: updatedAt, updated_by: updatedBy, ...rest } = packagePolicy; - - return rest; - }; - test('Should not dispatch if no policies', async () => { const context = buildManifestManagerContextMock({}); const manifestManager = new ManifestManager(context); @@ -807,7 +820,7 @@ describe('ManifestManager', () => { await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(0); }); test('Should return errors if invalid config for package policy', async () => { @@ -825,7 +838,7 @@ describe('ManifestManager', () => { new EndpointError(`Package Policy ${TEST_POLICY_ID_1} has no 'inputs[0].config'`), ]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(0); }); test('Should not dispatch if semantic version has not changed', async () => { @@ -854,7 +867,7 @@ describe('ManifestManager', () => { await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(0); }); test('Should dispatch to only policies where list of artifacts changed', async () => { @@ -898,17 +911,16 @@ describe('ManifestManager', () => { }, }), ]); - context.packagePolicyService.update = jest.fn().mockResolvedValue({}); + context.packagePolicyService.bulkUpdate = jest.fn().mockResolvedValue({}); await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(1); - expect(context.packagePolicyService.update).toHaveBeenNthCalledWith( + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(1); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenNthCalledWith( 1, expect.anything(), - undefined, - TEST_POLICY_ID_1, - toNewPackagePolicy( + context.esClient, + [ createPackagePolicyWithConfigMock({ id: TEST_POLICY_ID_1, config: { @@ -922,8 +934,8 @@ describe('ManifestManager', () => { }, }, }, - }) - ) + }), + ] ); }); @@ -970,17 +982,16 @@ describe('ManifestManager', () => { }, }), ]); - context.packagePolicyService.update = jest.fn().mockResolvedValue({}); + context.packagePolicyService.bulkUpdate = jest.fn().mockResolvedValue({}); await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(1); - expect(context.packagePolicyService.update).toHaveBeenNthCalledWith( + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(1); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenNthCalledWith( 1, expect.anything(), - undefined, - TEST_POLICY_ID_1, - toNewPackagePolicy( + context.esClient, + [ createPackagePolicyWithConfigMock({ id: TEST_POLICY_ID_1, config: { @@ -994,8 +1005,8 @@ describe('ManifestManager', () => { }, }, }, - }) - ) + }), + ] ); }); @@ -1033,17 +1044,13 @@ describe('ManifestManager', () => { }, }), ]); - context.packagePolicyService.update = jest.fn().mockImplementation(async (...args) => { - if (args[2] === TEST_POLICY_ID_2) { - throw error; - } else { - return {}; - } - }); + context.packagePolicyService.bulkUpdate = jest + .fn() + .mockResolvedValue({ updatedPolicies: [{}], failedPolicies: [{ error }] }); await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([error]); - expect(context.packagePolicyService.update).toHaveBeenCalledTimes(2); + expect(context.packagePolicyService.bulkUpdate).toHaveBeenCalledTimes(1); }); }); @@ -1074,9 +1081,9 @@ describe('ManifestManager', () => { const artifactToBeRemoved = await context.artifactClient.getArtifact(''); expect(artifactToBeRemoved).not.toBeUndefined(); - expect(context.artifactClient.deleteArtifact).toHaveBeenCalledWith( - getArtifactId(artifactToBeRemoved!) - ); + expect(context.artifactClient.bulkDeleteArtifacts).toHaveBeenCalledWith([ + getArtifactId(artifactToBeRemoved!), + ]); }); test('When there is no artifact to be removed', async () => { @@ -1113,7 +1120,7 @@ describe('ManifestManager', () => { await manifestManager.cleanup(manifest); - expect(context.artifactClient.deleteArtifact).toHaveBeenCalledTimes(0); + expect(context.artifactClient.bulkDeleteArtifacts).toHaveBeenCalledTimes(0); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 440e6366e262f..b7c98ef1b8773 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -7,17 +7,20 @@ import pMap from 'p-map'; import semver from 'semver'; -import { isEqual, isEmpty } from 'lodash'; +import { isEqual, isEmpty, chunk } from 'lodash'; +import type { ElasticsearchClient } from '@kbn/core/server'; import { type Logger, type SavedObjectsClientContract } from '@kbn/core/server'; import { ENDPOINT_EVENT_FILTERS_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID, ENDPOINT_BLOCKLISTS_LIST_ID, ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, + ENDPOINT_LIST_ID, } from '@kbn/securitysolution-list-constants'; -import type { ListResult } from '@kbn/fleet-plugin/common'; +import type { ListResult, PackagePolicy } from '@kbn/fleet-plugin/common'; import type { PackagePolicyClient } from '@kbn/fleet-plugin/server'; import type { ExceptionListClient } from '@kbn/lists-plugin/server'; +import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; import type { ManifestSchema } from '../../../../../common/endpoint/schema/manifest'; import { manifestDispatchSchema } from '../../../../../common/endpoint/schema/manifest'; @@ -26,9 +29,10 @@ import type { ArtifactListId } from '../../../lib/artifacts'; import { ArtifactConstants, buildArtifact, + getAllItemsFromEndpointExceptionList, getArtifactId, - getEndpointExceptionList, Manifest, + convertExceptionsToEndpointFormat, } from '../../../lib/artifacts'; import type { InternalArtifactCompleteSchema } from '../../../schemas/artifacts'; import { internalArtifactCompleteSchema } from '../../../schemas/artifacts'; @@ -49,36 +53,35 @@ interface BuildArtifactsForOsOptions { name: string; } -const iterateArtifactsBuildResult = async ( +const iterateArtifactsBuildResult = ( result: ArtifactsBuildResult, - callback: (artifact: InternalArtifactCompleteSchema, policyId?: string) => Promise + callback: (artifact: InternalArtifactCompleteSchema, policyId?: string) => void ) => { for (const artifact of result.defaultArtifacts) { - await callback(artifact); + callback(artifact); } for (const policyId of Object.keys(result.policySpecificArtifacts)) { for (const artifact of result.policySpecificArtifacts[policyId]) { - await callback(artifact, policyId); + callback(artifact, policyId); } } }; const iterateAllListItems = async ( - pageSupplier: (page: number) => Promise>, - itemCallback: (item: T) => void + pageSupplier: (page: number, perPage: number) => Promise>, + itemCallback: (items: T[]) => void ) => { let paging = true; let page = 1; + const perPage = 1000; while (paging) { - const { items, total } = await pageSupplier(page); + const { items, total } = await pageSupplier(page, perPage); - for (const item of items) { - await itemCallback(item); - } + itemCallback(items); - paging = (page - 1) * 20 + items.length < total; + paging = (page - 1) * perPage + items.length < total; page++; } }; @@ -90,6 +93,8 @@ export interface ManifestManagerContext { packagePolicyService: PackagePolicyClient; logger: Logger; experimentalFeatures: ExperimentalFeatures; + packagerTaskPackagePolicyUpdateBatchSize: number; + esClient: ElasticsearchClient; } const getArtifactIds = (manifest: ManifestSchema) => @@ -108,6 +113,9 @@ export class ManifestManager { protected logger: Logger; protected schemaVersion: ManifestSchemaVersion; protected experimentalFeatures: ExperimentalFeatures; + protected cachedExceptionsListsByOs: Map; + protected packagerTaskPackagePolicyUpdateBatchSize: number; + protected esClient: ElasticsearchClient; constructor(context: ManifestManagerContext) { this.artifactClient = context.artifactClient; @@ -117,6 +125,10 @@ export class ManifestManager { this.logger = context.logger; this.schemaVersion = 'v1'; this.experimentalFeatures = context.experimentalFeatures; + this.cachedExceptionsListsByOs = new Map(); + this.packagerTaskPackagePolicyUpdateBatchSize = + context.packagerTaskPackagePolicyUpdateBatchSize; + this.esClient = context.esClient; } /** @@ -129,45 +141,44 @@ export class ManifestManager { } /** - * Builds an artifact (one per supported OS) based on the current - * state of exception-list-agnostic SOs. + * Search or get exceptions from the cached map by listId and OS and filter those by policyId/global */ - protected async buildExceptionListArtifact(os: string): Promise { - return buildArtifact( - await getEndpointExceptionList({ - elClient: this.exceptionListClient, - schemaVersion: this.schemaVersion, + protected async getCachedExceptions({ + elClient, + listId, + os, + policyId, + schemaVersion, + }: { + elClient: ExceptionListClient; + listId: ArtifactListId; + os: string; + policyId?: string; + schemaVersion: string; + }) { + if (!this.cachedExceptionsListsByOs.has(`${listId}-${os}`)) { + const itemsByListId = await getAllItemsFromEndpointExceptionList({ + elClient, os, - }), - this.schemaVersion, - os, - ArtifactConstants.GLOBAL_ALLOWLIST_NAME - ); - } - - /** - * Builds an array of artifacts (one per supported OS) based on the current - * state of exception-list-agnostic SOs. - * - * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. - * @throws Throws/rejects if there are errors building the list. - */ - protected async buildExceptionListArtifacts(): Promise { - const defaultArtifacts: InternalArtifactCompleteSchema[] = []; - const policySpecificArtifacts: Record = {}; + listId, + }); + this.cachedExceptionsListsByOs.set(`${listId}-${os}`, itemsByListId); + } - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - defaultArtifacts.push(await this.buildExceptionListArtifact(os)); + const allExceptionsByListId = this.cachedExceptionsListsByOs.get(`${listId}-${os}`); + if (!allExceptionsByListId) { + throw new InvalidInternalManifestError(`Error getting exceptions for ${listId}-${os}`); } - await iterateAllListItems( - (page) => this.listEndpointPolicyIds(page), - async (policyId) => { - policySpecificArtifacts[policyId] = defaultArtifacts; - } - ); + const filter = (exception: ExceptionListItemSchema) => + policyId + ? exception.tags.includes('policy:all') || exception.tags.includes(`policy:${policyId}`) + : exception.tags.includes('policy:all'); - return { defaultArtifacts, policySpecificArtifacts }; + const exceptions: ExceptionListItemSchema[] = + listId === ENDPOINT_LIST_ID ? allExceptionsByListId : allExceptionsByListId.filter(filter); + + return convertExceptionsToEndpointFormat(exceptions, schemaVersion); } /** @@ -185,7 +196,7 @@ export class ManifestManager { policyId?: string; } & BuildArtifactsForOsOptions): Promise { return buildArtifact( - await getEndpointExceptionList({ + await this.getCachedExceptions({ elClient: this.exceptionListClient, schemaVersion: this.schemaVersion, os, @@ -198,13 +209,72 @@ export class ManifestManager { ); } + /** + * Builds an artifact (by policy) based on the current state of the + * artifacts list (Trusted Apps, Host Iso. Exceptions, Event Filters, Blocklists) + * (which uses the `exception-list-agnostic` SO type) + */ + protected async buildArtifactsByPolicy( + allPolicyIds: string[], + supportedOSs: string[], + osOptions: BuildArtifactsForOsOptions + ) { + const policySpecificArtifacts: Record = {}; + await pMap( + allPolicyIds, + async (policyId) => { + for (const os of supportedOSs) { + policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || []; + policySpecificArtifacts[policyId].push( + await this.buildArtifactsForOs({ os, policyId, ...osOptions }) + ); + } + }, + { + concurrency: 5, + /** When set to false, instead of stopping when a promise rejects, it will wait for all the promises to + * settle and then reject with an aggregated error containing all the errors from the rejected promises. */ + stopOnError: false, + } + ); + + return policySpecificArtifacts; + } + + /** + * Builds an array of artifacts (one per supported OS) based on the current + * state of exception-list-agnostic SOs. + * + * @returns {Promise} An array of uncompressed artifacts built from exception-list-agnostic SOs. + * @throws Throws/rejects if there are errors building the list. + */ + protected async buildExceptionListArtifacts( + allPolicyIds: string[] + ): Promise { + const defaultArtifacts: InternalArtifactCompleteSchema[] = []; + const policySpecificArtifacts: Record = {}; + const buildArtifactsForOsOptions: BuildArtifactsForOsOptions = { + listId: ENDPOINT_LIST_ID, + name: ArtifactConstants.GLOBAL_ALLOWLIST_NAME, + }; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + defaultArtifacts.push(await this.buildArtifactsForOs({ os, ...buildArtifactsForOsOptions })); + } + + allPolicyIds.forEach((policyId) => { + policySpecificArtifacts[policyId] = defaultArtifacts; + }); + + return { defaultArtifacts, policySpecificArtifacts }; + } + /** * Builds an array of artifacts (one per supported OS) based on the current state of the * Trusted Apps list (which uses the `exception-list-agnostic` SO type) */ - protected async buildTrustedAppsArtifacts(): Promise { + protected async buildTrustedAppsArtifacts(allPolicyIds: string[]): Promise { const defaultArtifacts: InternalArtifactCompleteSchema[] = []; - const policySpecificArtifacts: Record = {}; const buildArtifactsForOsOptions: BuildArtifactsForOsOptions = { listId: ENDPOINT_TRUSTED_APPS_LIST_ID, name: ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME, @@ -214,17 +284,12 @@ export class ManifestManager { defaultArtifacts.push(await this.buildArtifactsForOs({ os, ...buildArtifactsForOsOptions })); } - await iterateAllListItems( - (page) => this.listEndpointPolicyIds(page), - async (policyId) => { - for (const os of ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS) { - policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || []; - policySpecificArtifacts[policyId].push( - await this.buildArtifactsForOs({ os, policyId, ...buildArtifactsForOsOptions }) - ); - } - } - ); + const policySpecificArtifacts: Record = + await this.buildArtifactsByPolicy( + allPolicyIds, + ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS, + buildArtifactsForOsOptions + ); return { defaultArtifacts, policySpecificArtifacts }; } @@ -234,9 +299,10 @@ export class ManifestManager { * Event Filters list * @protected */ - protected async buildEventFiltersArtifacts(): Promise { + protected async buildEventFiltersArtifacts( + allPolicyIds: string[] + ): Promise { const defaultArtifacts: InternalArtifactCompleteSchema[] = []; - const policySpecificArtifacts: Record = {}; const buildArtifactsForOsOptions: BuildArtifactsForOsOptions = { listId: ENDPOINT_EVENT_FILTERS_LIST_ID, name: ArtifactConstants.GLOBAL_EVENT_FILTERS_NAME, @@ -246,17 +312,12 @@ export class ManifestManager { defaultArtifacts.push(await this.buildArtifactsForOs({ os, ...buildArtifactsForOsOptions })); } - await iterateAllListItems( - (page) => this.listEndpointPolicyIds(page), - async (policyId) => { - for (const os of ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS) { - policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || []; - policySpecificArtifacts[policyId].push( - await this.buildArtifactsForOs({ os, policyId, ...buildArtifactsForOsOptions }) - ); - } - } - ); + const policySpecificArtifacts: Record = + await this.buildArtifactsByPolicy( + allPolicyIds, + ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS, + buildArtifactsForOsOptions + ); return { defaultArtifacts, policySpecificArtifacts }; } @@ -266,29 +327,23 @@ export class ManifestManager { * Blocklist list * @protected */ - protected async buildBlocklistArtifacts(): Promise { + protected async buildBlocklistArtifacts(allPolicyIds: string[]): Promise { const defaultArtifacts: InternalArtifactCompleteSchema[] = []; - const policySpecificArtifacts: Record = {}; const buildArtifactsForOsOptions: BuildArtifactsForOsOptions = { listId: ENDPOINT_BLOCKLISTS_LIST_ID, name: ArtifactConstants.GLOBAL_BLOCKLISTS_NAME, }; - for (const os of ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS) { + for (const os of ArtifactConstants.SUPPORTED_BLOCKLISTS_OPERATING_SYSTEMS) { defaultArtifacts.push(await this.buildArtifactsForOs({ os, ...buildArtifactsForOsOptions })); } - await iterateAllListItems( - (page) => this.listEndpointPolicyIds(page), - async (policyId) => { - for (const os of ArtifactConstants.SUPPORTED_EVENT_FILTERS_OPERATING_SYSTEMS) { - policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || []; - policySpecificArtifacts[policyId].push( - await this.buildArtifactsForOs({ os, policyId, ...buildArtifactsForOsOptions }) - ); - } - } - ); + const policySpecificArtifacts: Record = + await this.buildArtifactsByPolicy( + allPolicyIds, + ArtifactConstants.SUPPORTED_BLOCKLISTS_OPERATING_SYSTEMS, + buildArtifactsForOsOptions + ); return { defaultArtifacts, policySpecificArtifacts }; } @@ -299,9 +354,10 @@ export class ManifestManager { * @returns */ - protected async buildHostIsolationExceptionsArtifacts(): Promise { + protected async buildHostIsolationExceptionsArtifacts( + allPolicyIds: string[] + ): Promise { const defaultArtifacts: InternalArtifactCompleteSchema[] = []; - const policySpecificArtifacts: Record = {}; const buildArtifactsForOsOptions: BuildArtifactsForOsOptions = { listId: ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID, name: ArtifactConstants.GLOBAL_HOST_ISOLATION_EXCEPTIONS_NAME, @@ -311,17 +367,12 @@ export class ManifestManager { defaultArtifacts.push(await this.buildArtifactsForOs({ os, ...buildArtifactsForOsOptions })); } - await iterateAllListItems( - (page) => this.listEndpointPolicyIds(page), - async (policyId) => { - for (const os of ArtifactConstants.SUPPORTED_HOST_ISOLATION_EXCEPTIONS_OPERATING_SYSTEMS) { - policySpecificArtifacts[policyId] = policySpecificArtifacts[policyId] || []; - policySpecificArtifacts[policyId].push( - await this.buildArtifactsForOs({ os, policyId, ...buildArtifactsForOsOptions }) - ); - } - } - ); + const policySpecificArtifacts: Record = + await this.buildArtifactsByPolicy( + allPolicyIds, + ArtifactConstants.SUPPORTED_HOST_ISOLATION_EXCEPTIONS_OPERATING_SYSTEMS, + buildArtifactsForOsOptions + ); return { defaultArtifacts, policySpecificArtifacts }; } @@ -385,16 +436,21 @@ export class ManifestManager { * @returns {Promise} Any errors encountered. */ public async deleteArtifacts(artifactIds: string[]): Promise { - const errors: Error[] = []; - for (const artifactId of artifactIds) { - try { - await this.artifactClient.deleteArtifact(artifactId); + try { + if (isEmpty(artifactIds)) { + return []; + } + const errors = await this.artifactClient.bulkDeleteArtifacts(artifactIds); + if (!isEmpty(errors)) { + return errors; + } + for (const artifactId of artifactIds) { this.logger.info(`Cleaned up artifact ${artifactId}`); - } catch (err) { - errors.push(err); } + return []; + } catch (err) { + return [err]; } - return errors; } /** @@ -461,14 +517,18 @@ export class ManifestManager { public async buildNewManifest( baselineManifest: Manifest = ManifestManager.createDefaultManifest(this.schemaVersion) ): Promise { + const allPolicyIds = await this.listEndpointPolicyIds(); const results = await Promise.all([ - this.buildExceptionListArtifacts(), - this.buildTrustedAppsArtifacts(), - this.buildEventFiltersArtifacts(), - this.buildHostIsolationExceptionsArtifacts(), - this.buildBlocklistArtifacts(), + this.buildExceptionListArtifacts(allPolicyIds), + this.buildTrustedAppsArtifacts(allPolicyIds), + this.buildEventFiltersArtifacts(allPolicyIds), + this.buildHostIsolationExceptionsArtifacts(allPolicyIds), + this.buildBlocklistArtifacts(allPolicyIds), ]); + // Clear cache as the ManifestManager instance is reused on every run. + this.cachedExceptionsListsByOs.clear(); + const manifest = new Manifest({ schemaVersion: this.schemaVersion, semanticVersion: baselineManifest.getSemanticVersion(), @@ -476,7 +536,7 @@ export class ManifestManager { }); for (const result of results) { - await iterateArtifactsBuildResult(result, async (artifact, policyId) => { + iterateArtifactsBuildResult(result, (artifact, policyId) => { const artifactToAdd = baselineManifest.getArtifact(getArtifactId(artifact)) || artifact; if (!internalArtifactCompleteSchema.is(artifactToAdd)) { throw new EndpointError( @@ -500,62 +560,83 @@ export class ManifestManager { * @returns {Promise} Any errors encountered. */ public async tryDispatch(manifest: Manifest): Promise { - const errors: Error[] = []; - + const allPackagePolicies: PackagePolicy[] = []; await iterateAllListItems( - (page) => this.listEndpointPolicies(page), - async (packagePolicy) => { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { id, revision, updated_at, updated_by, ...newPackagePolicy } = packagePolicy; - if (newPackagePolicy.inputs.length > 0 && newPackagePolicy.inputs[0].config !== undefined) { - const oldManifest = newPackagePolicy.inputs[0].config.artifact_manifest ?? { - value: {}, - }; - - const newManifestVersion = manifest.getSemanticVersion(); - if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { - const serializedManifest = manifest.toPackagePolicyManifest(packagePolicy.id); - - if (!manifestDispatchSchema.is(serializedManifest)) { - errors.push( - new EndpointError( - `Invalid manifest for policy ${packagePolicy.id}`, - serializedManifest - ) - ); - } else if (!manifestsEqual(serializedManifest, oldManifest.value)) { - newPackagePolicy.inputs[0].config.artifact_manifest = { value: serializedManifest }; - - try { - await this.packagePolicyService.update( - this.savedObjectsClient, - // @ts-expect-error TS2345 - undefined, - id, - newPackagePolicy - ); - this.logger.debug( - `Updated package policy ${id} with manifest version ${manifest.getSemanticVersion()}` - ); - } catch (err) { - errors.push(err); - } - } else { - this.logger.debug( - `No change in manifest content for package policy: ${id}. Staying on old version` - ); - } + (page, perPage) => this.listEndpointPolicies(page, perPage), + (packagePoliciesBatch) => { + allPackagePolicies.push(...packagePoliciesBatch); + } + ); + + const packagePoliciesToUpdate: PackagePolicy[] = []; + + const errors: Error[] = []; + allPackagePolicies.forEach((packagePolicy) => { + const { id } = packagePolicy; + if (packagePolicy.inputs.length > 0 && packagePolicy.inputs[0].config !== undefined) { + const oldManifest = packagePolicy.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + + const newManifestVersion = manifest.getSemanticVersion(); + if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { + const serializedManifest = manifest.toPackagePolicyManifest(id); + + if (!manifestDispatchSchema.is(serializedManifest)) { + errors.push(new EndpointError(`Invalid manifest for policy ${id}`, serializedManifest)); + } else if (!manifestsEqual(serializedManifest, oldManifest.value)) { + packagePolicy.inputs[0].config.artifact_manifest = { value: serializedManifest }; + packagePoliciesToUpdate.push(packagePolicy); } else { - this.logger.debug(`No change in manifest version for package policy: ${id}`); + this.logger.debug( + `No change in manifest content for package policy: ${id}. Staying on old version` + ); } } else { - errors.push( - new EndpointError(`Package Policy ${id} has no 'inputs[0].config'`, newPackagePolicy) - ); + this.logger.debug(`No change in manifest version for package policy: ${id}`); } + } else { + errors.push( + new EndpointError(`Package Policy ${id} has no 'inputs[0].config'`, packagePolicy) + ); } + }); + + // Split updates in batches with batch size: packagerTaskPackagePolicyUpdateBatchSize + const updateBatches = chunk( + packagePoliciesToUpdate, + this.packagerTaskPackagePolicyUpdateBatchSize ); + for (const currentBatch of updateBatches) { + const response = await this.packagePolicyService.bulkUpdate( + this.savedObjectsClient, + this.esClient, + currentBatch + ); + + // Parse errors + if (!isEmpty(response.failedPolicies)) { + errors.push( + ...response.failedPolicies.map((failedPolicy) => { + if (failedPolicy.error instanceof Error) { + return failedPolicy.error; + } else { + return new Error(failedPolicy.error.message); + } + }) + ); + } + // Log success updates + for (const updatedPolicy of response.updatedPolicies || []) { + this.logger.debug( + `Updated package policy ${ + updatedPolicy.id + } with manifest version ${manifest.getSemanticVersion()}` + ); + } + } + return errors; } @@ -583,20 +664,29 @@ export class ManifestManager { this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`); } - private async listEndpointPolicies(page: number) { + private async listEndpointPolicies(page: number, perPage: number) { return this.packagePolicyService.list(this.savedObjectsClient, { page, - perPage: 20, + perPage, kuery: 'ingest-package-policies.package.name:endpoint', }); } - private async listEndpointPolicyIds(page: number) { - return this.packagePolicyService.listIds(this.savedObjectsClient, { - page, - perPage: 20, - kuery: 'ingest-package-policies.package.name:endpoint', - }); + private async listEndpointPolicyIds() { + const allPolicyIds: string[] = []; + await iterateAllListItems( + (page, perPage) => { + return this.packagePolicyService.listIds(this.savedObjectsClient, { + page, + perPage, + kuery: 'ingest-package-policies.package.name:endpoint', + }); + }, + (packagePolicyIdsBatch) => { + allPolicyIds.push(...packagePolicyIdsBatch); + } + ); + return allPolicyIds; } public getArtifactsClient(): EndpointArtifactClientInterface { @@ -635,6 +725,7 @@ export class ManifestManager { } const badArtifacts = []; + const badArtifactIds = []; const manifestArtifactsIds = manifest .getAllArtifacts() @@ -646,6 +737,7 @@ export class ManifestManager { if (!isArtifactInManifest) { badArtifacts.push(fleetArtifact); + badArtifactIds.push(artifactId); } } @@ -657,16 +749,7 @@ export class ManifestManager { new EndpointError(`Cleaning up ${badArtifacts.length} orphan artifacts`, badArtifacts) ); - await pMap( - badArtifacts, - async (badArtifact) => this.artifactClient.deleteArtifact(getArtifactId(badArtifact)), - { - concurrency: 5, - /** When set to false, instead of stopping when a promise rejects, it will wait for all the promises to - * settle and then reject with an aggregated error containing all the errors from the rejected promises. */ - stopOnError: false, - } - ); + await this.artifactClient.bulkDeleteArtifacts(badArtifactIds); this.logger.info(`All orphan artifacts has been removed successfully`); } catch (error) { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts index 3d7b7b92f90c4..2acfa7b7b4794 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts @@ -62,6 +62,9 @@ export const createEndpointArtifactClientMock = ( listArtifacts: jest.fn((...args) => endpointArtifactClientMocked.listArtifacts(...args)), getArtifact: jest.fn((...args) => endpointArtifactClientMocked.getArtifact(...args)), deleteArtifact: jest.fn((...args) => endpointArtifactClientMocked.deleteArtifact(...args)), + bulkDeleteArtifacts: jest.fn(async (...args) => + endpointArtifactClientMocked.bulkDeleteArtifacts(...args) + ), _esClient: esClient, }; }; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f3a6ace3f2acd..badc35993ab03 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -459,6 +459,8 @@ export class Plugin implements ISecuritySolutionPlugin { packagePolicyService: plugins.fleet.packagePolicyService, logger, experimentalFeatures: config.experimentalFeatures, + packagerTaskPackagePolicyUpdateBatchSize: config.packagerTaskPackagePolicyUpdateBatchSize, + esClient: core.elasticsearch.client.asInternalUser, }); // Migrate artifacts to fleet and then start the minifest task after that is done