Skip to content

Commit

Permalink
Generates non sorted ES response at the end of the bulkCreateArtifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
dasansol92 committed Jun 28, 2023
1 parent af4805a commit 5e455b9
Showing 1 changed file with 13 additions and 18 deletions.
31 changes: 13 additions & 18 deletions x-pack/plugins/fleet/server/services/artifacts/artifacts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,20 +95,13 @@ export const BULK_CREATE_MAX_ARTIFACTS_BYTES = 4_000_000;
const generateArtifactBatches = (
artifacts: NewArtifact[],
maxArtifactsBatchSizeInBytes: number = BULK_CREATE_MAX_ARTIFACTS_BYTES
): {
batches: Array<Array<ArtifactElasticsearchProperties | { create: { _id: string } }>>;
nonSortedEsArtifactsResponse: Artifact[];
} => {
): Array<Array<ArtifactElasticsearchProperties | { create: { _id: string } }>> => {
const batches: Array<Array<ArtifactElasticsearchProperties | { create: { _id: string } }>> = [];
const nonSortedEsArtifactsResponse: Artifact[] = [];

let artifactsBatchLengthInBytes = 0;
const sortedArtifacts = sortBy(artifacts, 'encodedSize');

sortedArtifacts.forEach((artifact, index) => {
const nonSortedEsArtifactResponse = esSearchHitToArtifact({
_id: uniqueIdFromArtifact(artifacts[index]),
_source: newArtifactToElasticsearchProperties(artifacts[index]),
});
sortedArtifacts.forEach((artifact) => {
const esArtifact = newArtifactToElasticsearchProperties(artifact);
const bulkOperation = {
create: {
Expand All @@ -120,9 +113,6 @@ const generateArtifactBatches = (
// If there is no artifact yet added to the current batch, we add it anyway ignoring the batch limit as the batch size has to be > 0.
if (artifact.encodedSize + artifactsBatchLengthInBytes >= maxArtifactsBatchSizeInBytes) {
artifactsBatchLengthInBytes = artifact.encodedSize;
// Use non sorted artifacts array to preserve the artifacts order in the response
nonSortedEsArtifactsResponse.push(nonSortedEsArtifactResponse);

batches.push([bulkOperation, esArtifact]);
} else {
// Case it's the first one
Expand All @@ -131,22 +121,19 @@ const generateArtifactBatches = (
}
// Adds the next artifact to the current batch and increases the batch size count with the artifact size.
artifactsBatchLengthInBytes += artifact.encodedSize;
// Use non sorted artifacts array to preserve the artifacts order in the response
nonSortedEsArtifactsResponse.push(nonSortedEsArtifactResponse);

batches[batches.length - 1].push(bulkOperation, esArtifact);
}
});

return { batches, nonSortedEsArtifactsResponse };
return batches;
};

export const bulkCreateArtifacts = async (
esClient: ElasticsearchClient,
artifacts: NewArtifact[],
refresh = false
): Promise<{ artifacts?: Artifact[]; errors?: Error[] }> => {
const { batches, nonSortedEsArtifactsResponse } = generateArtifactBatches(
const batches = generateArtifactBatches(
artifacts,
appContextService.getConfig()?.createArtifactsBulkBatchSize
);
Expand Down Expand Up @@ -185,6 +172,14 @@ export const bulkCreateArtifacts = async (
return { errors: nonConflictErrors };
}

// Use non sorted artifacts array to preserve the artifacts order in the response
const nonSortedEsArtifactsResponse: Artifact[] = ([] = artifacts.map((artifact) => {
return esSearchHitToArtifact({
_id: uniqueIdFromArtifact(artifact),
_source: newArtifactToElasticsearchProperties(artifact),
});
}));

return {
artifacts: nonSortedEsArtifactsResponse,
};
Expand Down

0 comments on commit 5e455b9

Please sign in to comment.