diff --git a/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts b/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts index 1e51b0e37f1f8..1c54e543e7747 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts @@ -18,7 +18,7 @@ import { ArtifactsElasticsearchError } from '../../errors'; import { appContextService } from '../app_context'; import { createAppContextStartContractMock } from '../../mocks'; -import { newArtifactToElasticsearchProperties } from './mappings'; +import { newArtifactToElasticsearchProperties, uniqueIdFromArtifact } from './mappings'; import { generateArtifactEsGetSingleHitMock, @@ -186,14 +186,22 @@ describe('When using the artifacts services', () => { }); it('should create and return a multiple big artifacts', async () => { - const { ...generatedArtifact1 } = generateArtifactMock({ encodedSize: 5_000_500 }); - const newBigArtifact1 = generatedArtifact1; - const { ...generatedArtifact2 } = generateArtifactMock({ encodedSize: 500 }); - const newBigArtifact2 = generatedArtifact2; - const { ...generatedArtifact3 } = generateArtifactMock({ encodedSize: 233 }); - const newBigArtifact3 = generatedArtifact3; - const { ...generatedArtifact4 } = generateArtifactMock({ encodedSize: 7_000_000 }); - const newBigArtifact4 = generatedArtifact4; + const newBigArtifact1 = generateArtifactMock({ + encodedSize: 5_000_500, + decodedSha256: '1234', + }); + const newBigArtifact2 = generateArtifactMock({ + encodedSize: 500, + decodedSha256: '2345', + }); + const newBigArtifact3 = generateArtifactMock({ + encodedSize: 233, + decodedSha256: '3456', + }); + const newBigArtifact4 = generateArtifactMock({ + encodedSize: 7_000_000, + decodedSha256: '4567', + }); const { artifacts } = await bulkCreateArtifacts(esClientMock, [ newBigArtifact1, @@ -267,22 +275,22 @@ describe('When using the artifacts services', () => { expect(artifact1).toEqual({ ...newBigArtifact1, - id: expect.any(String), + id: uniqueIdFromArtifact(newBigArtifact1), created: expect.any(String), }); expect(artifact2).toEqual({ ...newBigArtifact2, - id: expect.any(String), + id: uniqueIdFromArtifact(newBigArtifact2), created: expect.any(String), }); expect(artifact3).toEqual({ ...newBigArtifact3, - id: expect.any(String), + id: uniqueIdFromArtifact(newBigArtifact3), created: expect.any(String), }); expect(artifact4).toEqual({ ...newBigArtifact4, - id: expect.any(String), + id: uniqueIdFromArtifact(newBigArtifact4), created: expect.any(String), }); }); diff --git a/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts b/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts index 2923b05a52cb8..0e312f142edf6 100644 --- a/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts +++ b/x-pack/plugins/fleet/server/services/artifacts/artifacts.ts @@ -95,20 +95,13 @@ export const BULK_CREATE_MAX_ARTIFACTS_BYTES = 4_000_000; const generateArtifactBatches = ( artifacts: NewArtifact[], maxArtifactsBatchSizeInBytes: number = BULK_CREATE_MAX_ARTIFACTS_BYTES -): { - batches: Array>; - artifactsEsResponse: Artifact[]; -} => { +): Array> => { const batches: Array> = []; - const artifactsEsResponse: Artifact[] = []; + let artifactsBatchLengthInBytes = 0; const sortedArtifacts = sortBy(artifacts, 'encodedSize'); - sortedArtifacts.forEach((artifact, index) => { - const esArtifactResponse = esSearchHitToArtifact({ - _id: uniqueIdFromArtifact(artifact), - _source: newArtifactToElasticsearchProperties(artifacts[index]), - }); + sortedArtifacts.forEach((artifact) => { const esArtifact = newArtifactToElasticsearchProperties(artifact); const bulkOperation = { create: { @@ -120,9 +113,6 @@ const generateArtifactBatches = ( // If there is no artifact yet added to the current batch, we add it anyway ignoring the batch limit as the batch size has to be > 0. if (artifact.encodedSize + artifactsBatchLengthInBytes >= maxArtifactsBatchSizeInBytes) { artifactsBatchLengthInBytes = artifact.encodedSize; - // Use non sorted artifacts array to preserve the artifacts order in the response - artifactsEsResponse.push(esArtifactResponse); - batches.push([bulkOperation, esArtifact]); } else { // Case it's the first one @@ -131,14 +121,11 @@ const generateArtifactBatches = ( } // Adds the next artifact to the current batch and increases the batch size count with the artifact size. artifactsBatchLengthInBytes += artifact.encodedSize; - // Use non sorted artifacts array to preserve the artifacts order in the response - artifactsEsResponse.push(esArtifactResponse); - batches[batches.length - 1].push(bulkOperation, esArtifact); } }); - return { batches, artifactsEsResponse }; + return batches; }; export const bulkCreateArtifacts = async ( @@ -146,7 +133,7 @@ export const bulkCreateArtifacts = async ( artifacts: NewArtifact[], refresh = false ): Promise<{ artifacts?: Artifact[]; errors?: Error[] }> => { - const { batches, artifactsEsResponse } = generateArtifactBatches( + const batches = generateArtifactBatches( artifacts, appContextService.getConfig()?.createArtifactsBulkBatchSize ); @@ -185,8 +172,16 @@ export const bulkCreateArtifacts = async ( return { errors: nonConflictErrors }; } + // Use non sorted artifacts array to preserve the artifacts order in the response + const nonSortedEsArtifactsResponse: Artifact[] = artifacts.map((artifact) => { + return esSearchHitToArtifact({ + _id: uniqueIdFromArtifact(artifact), + _source: newArtifactToElasticsearchProperties(artifact), + }); + }); + return { - artifacts: artifactsEsResponse, + artifacts: nonSortedEsArtifactsResponse, }; };