From e32153d842db14a49ab6dde398e9496ba7a7137e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 7 Sep 2021 14:43:26 -0400 Subject: [PATCH] [Security solution] [Endpoint] Fix bad artifact migration (#111294) (#111432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Working test that validate migrated artifact has same properties as SO artifact * Checks if artifact is compressed and uncompress it if necessary before creating the new one from fleet Co-authored-by: Paul Tavares Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: David Sánchez Co-authored-by: Paul Tavares --- .../fleet/server/services/artifacts/mocks.ts | 13 ++++- .../migrate_artifacts_to_fleet.test.ts | 56 ++++++++++++++++++- .../artifacts/migrate_artifacts_to_fleet.ts | 19 ++++++- .../endpoint/services/artifacts/mocks.ts | 36 ++++++++++-- 4 files changed, 113 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/artifacts/mocks.ts b/x-pack/plugins/fleet/server/services/artifacts/mocks.ts index 8bc1768da23a2..2799e1807123d 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/mocks.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/mocks.ts @@ -12,7 +12,12 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks'; import type { SearchHit, ESSearchResponse } from '../../../../../../src/core/types/elasticsearch'; -import type { Artifact, ArtifactElasticsearchProperties, ArtifactsClientInterface } from './types'; +import type { + Artifact, + ArtifactElasticsearchProperties, + ArtifactsClientInterface, + NewArtifact, +} from './types'; import { newArtifactToElasticsearchProperties } from './mappings'; export const createArtifactsClientMock = (): jest.Mocked => { @@ -77,10 +82,12 @@ export const generateEsRequestErrorApiResponseMock = ( ); }; -export const generateArtifactEsGetSingleHitMock = (): SearchHit => { +export const generateArtifactEsGetSingleHitMock = ( + artifact?: NewArtifact +): SearchHit => { const { id, created, ...newArtifact } = generateArtifactMock(); const _source = { - ...newArtifactToElasticsearchProperties(newArtifact), + ...newArtifactToElasticsearchProperties(artifact ?? newArtifact), created, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts index 2071d4b8c27b7..d6599f2686670 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts @@ -15,14 +15,20 @@ import { SavedObjectsFindResponse, SavedObjectsFindResult, } from 'kibana/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { migrateArtifactsToFleet } from './migrate_artifacts_to_fleet'; import { createEndpointArtifactClientMock } from '../../services/artifacts/mocks'; -import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { InternalArtifactCompleteSchema } from '../../schemas'; +import { generateArtifactEsGetSingleHitMock } from '../../../../../fleet/server/services/artifacts/mocks'; +import { NewArtifact } from '../../../../../fleet/server/services'; +import { CreateRequest } from '@elastic/elasticsearch/api/types'; describe('When migrating artifacts to fleet', () => { let soClient: jest.Mocked; let logger: jest.Mocked; let artifactClient: ReturnType; + /** An artifact that was created prior to 7.14 */ + let soArtifactEntry: InternalArtifactCompleteSchema; const createSoFindResult = ( soHits: SavedObjectsFindResult[] = [], @@ -41,6 +47,41 @@ describe('When migrating artifacts to fleet', () => { soClient = savedObjectsClientMock.create() as jest.Mocked; logger = loggingSystemMock.create().get() as jest.Mocked; artifactClient = createEndpointArtifactClientMock(); + // pre-v7.14 artifact, which is compressed + soArtifactEntry = { + identifier: 'endpoint-exceptionlist-macos-v1', + compressionAlgorithm: 'zlib', + encryptionAlgorithm: 'none', + decodedSha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encodedSha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decodedSize: 14, + encodedSize: 22, + body: 'eJyrVkrNKynKTC1WsoqOrQUAJxkFKQ==', + }; + + // Mock the esClient create response to include the artifact properties that were provide + // to it by fleet artifact client + artifactClient._esClient.create.mockImplementation((props: CreateRequest) => { + return elasticsearchServiceMock.createSuccessTransportRequestPromise({ + ...generateArtifactEsGetSingleHitMock({ + ...((props?.body ?? {}) as NewArtifact), + }), + _index: '.fleet-artifacts-7', + _id: `endpoint:endpoint-exceptionlist-macos-v1-${ + // @ts-ignore + props?.body?.decodedSha256 ?? 'UNKNOWN?' + }`, + _version: 1, + result: 'created', + _shards: { + total: 1, + successful: 1, + failed: 0, + }, + _seq_no: 0, + _primary_term: 1, + }); + }); soClient.find.mockResolvedValue(createSoFindResult([], 0)).mockResolvedValueOnce( createSoFindResult([ @@ -49,7 +90,7 @@ describe('When migrating artifacts to fleet', () => { type: '', id: 'abc123', references: [], - attributes: await getInternalArtifactMock('windows', 'v1'), + attributes: soArtifactEntry, }, ]) ); @@ -70,6 +111,17 @@ describe('When migrating artifacts to fleet', () => { expect(soClient.delete).toHaveBeenCalled(); }); + it('should create artifact in fleet with attributes that match the SO version', async () => { + await migrateArtifactsToFleet(soClient, artifactClient, logger); + + await expect(artifactClient.createArtifact.mock.results[0].value).resolves.toEqual( + expect.objectContaining({ + ...soArtifactEntry, + compressionAlgorithm: 'zlib', + }) + ); + }); + it('should ignore 404 responses for SO delete (multi-node kibana setup)', async () => { const notFoundError: Error & { output?: { statusCode: number } } = new Error('not found'); notFoundError.output = { statusCode: 404 }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts index 4518e23bb7fea..07edfce24affd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts @@ -5,9 +5,11 @@ * 2.0. */ +import { inflate as _inflate } from 'zlib'; +import { promisify } from 'util'; import { SavedObjectsClient, Logger } from 'kibana/server'; import { EndpointArtifactClientInterface } from '../../services'; -import { InternalArtifactCompleteSchema } from '../../schemas'; +import { InternalArtifactCompleteSchema, InternalArtifactSchema } from '../../schemas'; import { ArtifactConstants } from './common'; class ArtifactMigrationError extends Error { @@ -16,6 +18,12 @@ class ArtifactMigrationError extends Error { } } +const inflateAsync = promisify(_inflate); + +function isCompressed(artifact: InternalArtifactSchema) { + return artifact.compressionAlgorithm === 'zlib'; +} + /** * With v7.13, artifact storage was moved from a security_solution saved object to a fleet index * in order to support Fleet Server. @@ -57,6 +65,15 @@ export const migrateArtifactsToFleet = async ( } for (const artifact of artifactList) { + if (isCompressed(artifact.attributes)) { + artifact.attributes = { + ...artifact.attributes, + body: (await inflateAsync(Buffer.from(artifact.attributes.body, 'base64'))).toString( + 'base64' + ), + }; + } + // Create new artifact in fleet index await endpointArtifactClient.createArtifact(artifact.attributes); // Delete old artifact from SO and if there are errors here, then ignore 404's diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts index 18be9f299c15c..e2a4f9a3f5356 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts @@ -8,7 +8,14 @@ import { SavedObjectsClientContract } from 'src/core/server'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ManifestClient } from './manifest_client'; -import { EndpointArtifactClientInterface } from './artifact_client'; +import { EndpointArtifactClient, EndpointArtifactClientInterface } from './artifact_client'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { ElasticsearchClientMock } from '../../../../../../../src/core/server/elasticsearch/client/mocks'; +import { elasticsearchServiceMock } from '../../../../../../../src/core/server/mocks'; +// Because mocks are for testing only, should be ok to import the FleetArtifactsClient directly +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { FleetArtifactsClient } from '../../../../../fleet/server/services'; +import { createArtifactsClientMock } from '../../../../../fleet/server/mocks'; export const getManifestClientMock = ( savedObjectsClient?: SavedObjectsClientContract @@ -19,10 +26,29 @@ export const getManifestClientMock = ( return new ManifestClient(savedObjectsClientMock.create(), 'v1'); }; -export const createEndpointArtifactClientMock = (): jest.Mocked => { +/** + * Returns back a mocked EndpointArtifactClient along with the internal FleetArtifactsClient and the Es Clients that are being used + * @param esClient + */ +export const createEndpointArtifactClientMock = ( + esClient: ElasticsearchClientMock = elasticsearchServiceMock.createScopedClusterClient() + .asInternalUser +): jest.Mocked & { + _esClient: ElasticsearchClientMock; +} => { + const fleetArtifactClientMocked = createArtifactsClientMock(); + const endpointArtifactClientMocked = new EndpointArtifactClient(fleetArtifactClientMocked); + + // Return the interface mocked with jest.fn() that fowards calls to the real instance return { - createArtifact: jest.fn(), - getArtifact: jest.fn(), - deleteArtifact: jest.fn(), + createArtifact: jest.fn(async (...args) => { + const fleetArtifactClient = new FleetArtifactsClient(esClient, 'endpoint'); + const endpointArtifactClient = new EndpointArtifactClient(fleetArtifactClient); + const response = await endpointArtifactClient.createArtifact(...args); + return response; + }), + getArtifact: jest.fn((...args) => endpointArtifactClientMocked.getArtifact(...args)), + deleteArtifact: jest.fn((...args) => endpointArtifactClientMocked.deleteArtifact(...args)), + _esClient: esClient, }; };