From 66fd6eb0effad25c290e90e376addec0738339dd Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Mon, 3 Jul 2023 09:00:36 -0400 Subject: [PATCH] [Fleet] Update fleet file storage indices to the new Datastream names (#160998) ## Summary - Updates fleet file storage indexes to the new Datastream names: | Old Name | New Name | |-----------|-------------------| | `.fleet-file-data-*` | `.fleet-fileds-fromhost-data-*` | | `.fleet-files-*` | `.fleet-fileds-fromhost-meta-*` | | `.fleet-filedelivery-data-*` | `.fleet-fileds-tohost-data-*` | | `.fleet-filedelivery-meta-*` | `.fleet-fileds-tohost-meta-*` | - Removes code that was initializing the old backing indexes - Updates the `fleet:check-deleted-files-task` to ensure it correctly parses the index name/alias from the underlying chunk backing index - Update Security Solution dev scripts, types and mocks to include the `@timestamp` property and ensure any mocks indexed use `op_type:create` --- .../fleet/common/constants/file_storage.ts | 14 +-- .../common/services/file_storage.test.ts | 25 +++- .../fleet/common/services/file_storage.ts | 28 ++--- .../epm/elasticsearch/template/install.ts | 66 ----------- .../services/epm/packages/_install_package.ts | 10 -- .../services/files/client_from_host.test.ts | 10 +- .../services/files/client_to_host.test.ts | 4 +- .../fleet/server/services/files/index.ts | 111 ++++++++++-------- .../fleet/server/services/files/mocks.ts | 2 +- .../fleet/server/services/setup.test.ts | 14 +-- x-pack/plugins/fleet/server/services/setup.ts | 39 +----- .../server/tasks/check_deleted_files_task.ts | 28 ++++- .../common/endpoint/types/actions.ts | 1 + .../endpoint/common/response_actions.ts | 5 +- .../scripts/endpoint/common/stack_services.ts | 2 +- .../server/endpoint/services/actions/mocks.ts | 1 + .../apis/agents/uploads.ts | 73 ++++++++---- .../apis/file_upload_index.ts | 34 ------ .../apis/index.ts | 1 - 19 files changed, 187 insertions(+), 281 deletions(-) delete mode 100644 x-pack/test/security_solution_endpoint_api_int/apis/file_upload_index.ts diff --git a/x-pack/plugins/fleet/common/constants/file_storage.ts b/x-pack/plugins/fleet/common/constants/file_storage.ts index 554176f40e263..0d796d691b97b 100644 --- a/x-pack/plugins/fleet/common/constants/file_storage.ts +++ b/x-pack/plugins/fleet/common/constants/file_storage.ts @@ -8,26 +8,26 @@ // File storage indexes supporting file upload from the host to Elastic/Kibana // If needing to get an integration specific index name, use the utility functions // found in `common/services/file_storage` -export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-files-*'; -export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-file-data-*'; +export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-fileds-fromhost-meta-*'; +export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-fileds-fromhost-data-*'; -// File storage indexes supporting user uplaoded files (via kibana) that will be +// File storage indexes supporting user uploaded files (via kibana) that will be // delivered to the host agent/endpoint -export const FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN = '.fleet-filedelivery-meta-*'; -export const FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN = '.fleet-filedelivery-data-*'; +export const FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN = '.fleet-fileds-tohost-meta-*'; +export const FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN = '.fleet-fileds-tohost-data-*'; // which integrations support file upload and the name to use for the file upload index export const FILE_STORAGE_INTEGRATION_INDEX_NAMES: Readonly< Record< string, - { + Readonly<{ /** name to be used for the index */ name: string; /** If integration supports files sent from host to ES/Kibana */ fromHost: boolean; /** If integration supports files to be sent to host from kibana */ toHost: boolean; - } + }> > > = { elastic_agent: { name: 'agent', fromHost: true, toHost: false }, diff --git a/x-pack/plugins/fleet/common/services/file_storage.test.ts b/x-pack/plugins/fleet/common/services/file_storage.test.ts index 0360d7311eb6a..dbf5da61dba1d 100644 --- a/x-pack/plugins/fleet/common/services/file_storage.test.ts +++ b/x-pack/plugins/fleet/common/services/file_storage.test.ts @@ -5,24 +5,41 @@ * 2.0. */ +import { FILE_STORAGE_METADATA_INDEX_PATTERN } from '../constants'; + import { getFileDataIndexName, getFileMetadataIndexName } from '..'; +import { getIntegrationNameFromIndexName } from './file_storage'; + describe('File Storage services', () => { describe('File Index Names', () => { it('should generate file metadata index name for files received from host', () => { - expect(getFileMetadataIndexName('foo')).toEqual('.fleet-files-foo'); + expect(getFileMetadataIndexName('foo')).toEqual('.fleet-fileds-fromhost-meta-foo'); }); it('should generate file data index name for files received from host', () => { - expect(getFileDataIndexName('foo')).toEqual('.fleet-file-data-foo'); + expect(getFileDataIndexName('foo')).toEqual('.fleet-fileds-fromhost-data-foo'); }); it('should generate file metadata index name for files to be delivered to host', () => { - expect(getFileMetadataIndexName('foo', true)).toEqual('.fleet-filedelivery-meta-foo'); + expect(getFileMetadataIndexName('foo', true)).toEqual('.fleet-fileds-tohost-meta-foo'); }); it('should generate file data index name for files to be delivered to host', () => { - expect(getFileDataIndexName('foo', true)).toEqual('.fleet-filedelivery-data-foo'); + expect(getFileDataIndexName('foo', true)).toEqual('.fleet-fileds-tohost-data-foo'); + }); + }); + + describe('getIntegrationNameFromIndexName()', () => { + it.each([ + ['regular index names', '.fleet-fileds-fromhost-meta-agent'], + ['datastream index names', '.ds-.fleet-fileds-fromhost-data-agent-2023.06.30-00001'], + ])('should handle %s', (_, index) => { + expect(getIntegrationNameFromIndexName(index, FILE_STORAGE_METADATA_INDEX_PATTERN)).toEqual( + 'agent' + ); }); + + it.todo('should error if index pattern does not include `*`'); }); }); diff --git a/x-pack/plugins/fleet/common/services/file_storage.ts b/x-pack/plugins/fleet/common/services/file_storage.ts index 6581f671df663..af909a22aa946 100644 --- a/x-pack/plugins/fleet/common/services/file_storage.ts +++ b/x-pack/plugins/fleet/common/services/file_storage.ts @@ -56,21 +56,19 @@ export const getFileDataIndexName = ( ); }; -/** - * Returns the write index name for a given file upload alias name, this is the same for metadata and chunks - * @param aliasName - */ -export const getFileWriteIndexName = (aliasName: string) => aliasName + '-000001'; /** * Returns back the integration name for a given File Data (chunks) index name. * * @example - * // Given a File data index pattern of `.fleet-file-data-*`: + * // Given a File data index pattern of `.fleet-fileds-fromhost-data-*`: * - * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent'); + * getIntegrationNameFromFileDataIndexName('.fleet-fileds-fromhost-data-agent'); * // return 'agent' * - * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent-00001'); + * getIntegrationNameFromFileDataIndexName('.ds-.fleet-fileds-fromhost-data-agent'); + * // return 'agent' + * + * getIntegrationNameFromFileDataIndexName('.ds-.fleet-fileds-fromhost-data-agent-2023.06.30-00001'); * // return 'agent' */ export const getIntegrationNameFromFileDataIndexName = (indexName: string): string => { @@ -87,7 +85,7 @@ export const getIntegrationNameFromIndexName = ( throw new Error(`Unable to parse index name. No '*' in index pattern: ${indexPattern}`); } - const indexPieces = indexName.split('-'); + const indexPieces = indexName.replace(/^\.ds-/, '').split('-'); if (indexPieces[integrationNameIndexPosition]) { return indexPieces[integrationNameIndexPosition]; @@ -95,15 +93,3 @@ export const getIntegrationNameFromIndexName = ( throw new Error(`Index name ${indexName} does not seem to be a File storage index`); }; - -export const getFileStorageWriteIndexBody = (aliasName: string) => ({ - aliases: { - [aliasName]: { - is_write_index: true, - }, - }, - settings: { - 'index.lifecycle.rollover_alias': aliasName, - 'index.hidden': true, - }, -}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index f5ef34d28b5c4..1780d3e55169a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -11,18 +11,9 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { - FILE_STORAGE_INTEGRATION_INDEX_NAMES, - FILE_STORAGE_INTEGRATION_NAMES, -} from '../../../../../common/constants'; - import { ElasticsearchAssetType } from '../../../../types'; import { - getFileWriteIndexName, - getFileStorageWriteIndexBody, getPipelineNameForDatastream, - getFileDataIndexName, - getFileMetadataIndexName, getRegistryDataStreamAssetBaseName, } from '../../../../../common/services'; import type { @@ -440,63 +431,6 @@ export async function ensureDefaultComponentTemplates( ); } -/* - * Given a list of integration names, if the integrations support file upload - * then ensure that the alias has a matching write index, as we use "plain" indices - * not data streams. - * e.g .fleet-file-data-agent must have .fleet-file-data-agent-00001 as the write index - * before files can be uploaded. - */ -export async function ensureFileUploadWriteIndices(opts: { - esClient: ElasticsearchClient; - logger: Logger; - integrationNames: string[]; -}) { - const { esClient, logger, integrationNames } = opts; - - const integrationsWithFileUpload = integrationNames.filter((integration) => - FILE_STORAGE_INTEGRATION_NAMES.includes(integration as any) - ); - - if (!integrationsWithFileUpload.length) return []; - - const ensure = (aliasName: string) => - ensureAliasHasWriteIndex({ - esClient, - logger, - aliasName, - writeIndexName: getFileWriteIndexName(aliasName), - body: getFileStorageWriteIndexBody(aliasName), - }); - - return Promise.all( - integrationsWithFileUpload.flatMap((integrationName) => { - const { - name: indexName, - fromHost, - toHost, - } = FILE_STORAGE_INTEGRATION_INDEX_NAMES[integrationName]; - const indexCreateRequests: Array> = []; - - if (fromHost) { - indexCreateRequests.push( - ensure(getFileDataIndexName(indexName)), - ensure(getFileMetadataIndexName(indexName)) - ); - } - - if (toHost) { - indexCreateRequests.push( - ensure(getFileDataIndexName(indexName, true)), - ensure(getFileMetadataIndexName(indexName, true)) - ); - } - - return indexCreateRequests; - }) - ); -} - export async function ensureComponentTemplate( esClient: ElasticsearchClient, logger: Logger, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index b884e8c893de8..a5b4ad6f4e00b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -37,7 +37,6 @@ import type { PackageVerificationResult, IndexTemplateEntry, } from '../../../types'; -import { ensureFileUploadWriteIndices } from '../elasticsearch/template/install'; import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; import { isTopLevelPipeline, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline'; import { installILMPolicy } from '../elasticsearch/ilm/install'; @@ -236,15 +235,6 @@ export async function _installPackage({ logger.warn(`Error removing legacy templates: ${e.message}`); } - const { diagnosticFileUploadEnabled } = appContextService.getExperimentalFeatures(); - if (diagnosticFileUploadEnabled) { - await ensureFileUploadWriteIndices({ - integrationNames: [packageInfo.name], - esClient, - logger, - }); - } - // update current backing indices of each data stream await withPackageSpan('Update write indices', () => updateCurrentWriteIndices(esClient, logger, indexTemplates) diff --git a/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts b/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts index 068bad018b5e8..0051418b1c00d 100644 --- a/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts +++ b/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts @@ -91,11 +91,11 @@ describe('FleetFromHostFilesClient', () => { esClientMock.search.mockImplementation(async (searchRequest = {}) => { // File metadata - if ((searchRequest.index as string).startsWith('.fleet-files-')) { + if ((searchRequest.index as string).startsWith('.fleet-fileds-fromhost-meta-')) { return fleetFilesIndexSearchResponse; } - if ((searchRequest.index as string).startsWith('.fleet-file-data-')) { + if ((searchRequest.index as string).startsWith('.fleet-fileds-fromhost-data-')) { return fleetFileDataIndexSearchResponse; } @@ -111,8 +111,8 @@ describe('FleetFromHostFilesClient', () => { expect(createEsFileClientMock).toHaveBeenCalledWith({ elasticsearchClient: esClientMock, logger: loggerMock, - metadataIndex: '.fleet-files-foo', - blobStorageIndex: '.fleet-file-data-foo', + metadataIndex: '.fleet-fileds-fromhost-meta-foo', + blobStorageIndex: '.fleet-fileds-fromhost-data-foo', indexIsAlias: true, }); }); @@ -159,7 +159,7 @@ describe('FleetFromHostFilesClient', () => { }, }, }, - index: '.fleet-file-data-foo', + index: '.fleet-fileds-fromhost-data-foo', size: 0, }); }); diff --git a/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts b/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts index a4820f256da95..1068f34366970 100644 --- a/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts +++ b/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts @@ -130,8 +130,8 @@ describe('FleetToHostFilesClient', () => { expect(createEsFileClientMock).toHaveBeenCalledWith({ elasticsearchClient: esClientMock, logger: loggerMock, - metadataIndex: '.fleet-filedelivery-meta-foo', - blobStorageIndex: '.fleet-filedelivery-data-foo', + metadataIndex: '.fleet-fileds-tohost-meta-foo', + blobStorageIndex: '.fleet-fileds-tohost-data-foo', maxSizeBytes: 12345, indexIsAlias: true, }); diff --git a/x-pack/plugins/fleet/server/services/files/index.ts b/x-pack/plugins/fleet/server/services/files/index.ts index 5790164b81d07..8d6cbdb9fd5a4 100644 --- a/x-pack/plugins/fleet/server/services/files/index.ts +++ b/x-pack/plugins/fleet/server/services/files/index.ts @@ -34,22 +34,27 @@ export async function getFilesByStatus( abortController: AbortController, status: FileStatus = 'READY' ): Promise { - const result = await esClient.search( - { - index: FILE_STORAGE_METADATA_INDEX_PATTERN, - body: { - size: ES_SEARCH_LIMIT, - query: { - term: { - 'file.Status': status, + const result = await esClient + .search( + { + index: FILE_STORAGE_METADATA_INDEX_PATTERN, + body: { + size: ES_SEARCH_LIMIT, + query: { + term: { + 'file.Status': status, + }, }, + _source: false, }, - _source: false, + ignore_unavailable: true, }, - ignore_unavailable: true, - }, - { signal: abortController.signal } - ); + { signal: abortController.signal } + ) + .catch((err) => { + Error.captureStackTrace(err); + throw err; + }); return result.hits.hits; } @@ -84,32 +89,37 @@ export async function fileIdsWithoutChunksByIndex( return acc; }, {} as FileIdsByIndex); - const chunks = await esClient.search<{ bid: string }>( - { - index: FILE_STORAGE_DATA_INDEX_PATTERN, - body: { - size: ES_SEARCH_LIMIT, - query: { - bool: { - must: [ - { - terms: { - bid: Array.from(allFileIds), + const chunks = await esClient + .search<{ bid: string }>( + { + index: FILE_STORAGE_DATA_INDEX_PATTERN, + body: { + size: ES_SEARCH_LIMIT, + query: { + bool: { + must: [ + { + terms: { + bid: Array.from(allFileIds), + }, }, - }, - { - term: { - last: true, + { + term: { + last: true, + }, }, - }, - ], + ], + }, }, + _source: ['bid'], }, - _source: ['bid'], }, - }, - { signal: abortController.signal } - ); + { signal: abortController.signal } + ) + .catch((err) => { + Error.captureStackTrace(err); + throw err; + }); chunks.hits.hits.forEach((hit) => { const fileId = hit._source?.bid; @@ -140,22 +150,27 @@ export function updateFilesStatus( ): Promise { return Promise.all( Object.entries(fileIdsByIndex).map(([index, fileIds]) => { - return esClient.updateByQuery( - { - index, - refresh: true, - query: { - ids: { - values: Array.from(fileIds), + return esClient + .updateByQuery( + { + index, + refresh: true, + query: { + ids: { + values: Array.from(fileIds), + }, + }, + script: { + source: `ctx._source.file.Status = '${status}'`, + lang: 'painless', }, }, - script: { - source: `ctx._source.file.Status = '${status}'`, - lang: 'painless', - }, - }, - { signal: abortController.signal } - ); + { signal: abortController.signal } + ) + .catch((err) => { + Error.captureStackTrace(err); + throw err; + }); }) ); } diff --git a/x-pack/plugins/fleet/server/services/files/mocks.ts b/x-pack/plugins/fleet/server/services/files/mocks.ts index 35c276bf5cae7..23c0482b7e111 100644 --- a/x-pack/plugins/fleet/server/services/files/mocks.ts +++ b/x-pack/plugins/fleet/server/services/files/mocks.ts @@ -86,7 +86,7 @@ export const createFromHostEsSearchResponseMock = max_score: 0, hits: [ { - _index: '.fleet-files-foo-000001', + _index: '.fleet-fileds-fromhost-meta-foo-000001', _id: '123', _score: 1.0, _source: { diff --git a/x-pack/plugins/fleet/server/services/setup.test.ts b/x-pack/plugins/fleet/server/services/setup.test.ts index 7d98db879910b..15dccb15053ab 100644 --- a/x-pack/plugins/fleet/server/services/setup.test.ts +++ b/x-pack/plugins/fleet/server/services/setup.test.ts @@ -15,9 +15,7 @@ import { ensurePreconfiguredPackagesAndPolicies } from '.'; import { appContextService } from './app_context'; import { getInstallations } from './epm/packages'; import { upgradeManagedPackagePolicies } from './managed_package_policies'; -import { setupFleet, ensureFleetFileUploadIndices } from './setup'; - -import { ensureFileUploadWriteIndices } from './epm/elasticsearch/template/install'; +import { setupFleet } from './setup'; jest.mock('./preconfiguration'); jest.mock('./preconfiguration/outputs'); @@ -70,8 +68,6 @@ describe('setupFleet', () => { soClient.find.mockResolvedValue({ saved_objects: [] } as any); soClient.bulkGet.mockResolvedValue({ saved_objects: [] } as any); - - (ensureFileUploadWriteIndices as jest.Mock).mockResolvedValue({}); }); afterEach(async () => { @@ -138,12 +134,4 @@ describe('setupFleet', () => { ], }); }); - - it('should create agent file upload write indices', async () => { - await ensureFleetFileUploadIndices(soClient, esClient); - - expect((ensureFileUploadWriteIndices as jest.Mock).mock.calls[0][0].integrationNames).toEqual([ - 'elastic_agent', - ]); - }); }); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 90b76b8495e39..92d8f8fb37dda 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -12,11 +12,7 @@ import pMap from 'p-map'; import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; -import { - AUTO_UPDATE_PACKAGES, - FILE_STORAGE_INTEGRATION_NAMES, - FLEET_ELASTIC_AGENT_PACKAGE, -} from '../../common/constants'; +import { AUTO_UPDATE_PACKAGES } from '../../common/constants'; import type { PreconfigurationError } from '../../common/constants'; import type { DefaultPackagesInstallationError, @@ -44,10 +40,7 @@ import { ensureDefaultEnrollmentAPIKeyForAgentPolicy } from './api_keys'; import { getRegistryUrl, settingsService } from '.'; import { awaitIfPending } from './setup_utils'; import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; -import { - ensureDefaultComponentTemplates, - ensureFileUploadWriteIndices, -} from './epm/elasticsearch/template/install'; +import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; @@ -60,7 +53,6 @@ import { ensurePreconfiguredFleetServerHosts, getPreconfiguredFleetServerHostFromConfig, } from './preconfiguration/fleet_server_host'; -import { getInstallationsByName } from './epm/packages/get'; export interface SetupStatus { isInitialized: boolean; @@ -125,7 +117,6 @@ async function createSetupSideEffects( logger.debug('Setting up Fleet Elasticsearch assets'); await ensureFleetGlobalEsAssets(soClient, esClient); - await ensureFleetFileUploadIndices(soClient, esClient); // Ensure that required packages are always installed even if they're left out of the config const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name)); @@ -207,32 +198,6 @@ async function createSetupSideEffects( }; } -/** - * Ensure ES assets shared by all Fleet index template are installed - */ -export async function ensureFleetFileUploadIndices( - soClient: SavedObjectsClientContract, - esClient: ElasticsearchClient -) { - const { diagnosticFileUploadEnabled } = appContextService.getExperimentalFeatures(); - if (!diagnosticFileUploadEnabled) return; - const logger = appContextService.getLogger(); - const installedFileUploadIntegrations = await getInstallationsByName({ - savedObjectsClient: soClient, - pkgNames: [...FILE_STORAGE_INTEGRATION_NAMES], - }); - - const integrationNames = installedFileUploadIntegrations.map(({ name }) => name); - if (!integrationNames.includes(FLEET_ELASTIC_AGENT_PACKAGE)) { - integrationNames.push(FLEET_ELASTIC_AGENT_PACKAGE); - } - logger.debug(`Ensuring file upload write indices for ${integrationNames}`); - return ensureFileUploadWriteIndices({ - esClient, - logger, - integrationNames, - }); -} /** * Ensure ES assets shared by all Fleet index template are installed */ diff --git a/x-pack/plugins/fleet/server/tasks/check_deleted_files_task.ts b/x-pack/plugins/fleet/server/tasks/check_deleted_files_task.ts index b7ebdd0748e9d..a7611d73cd313 100644 --- a/x-pack/plugins/fleet/server/tasks/check_deleted_files_task.ts +++ b/x-pack/plugins/fleet/server/tasks/check_deleted_files_task.ts @@ -72,6 +72,7 @@ export class CheckDeletedFilesTask { } this.wasStarted = true; + this.logger.info(`Started with interval of [${INTERVAL}] and timeout of [${TIMEOUT}]`); try { await taskManager.ensureScheduled({ @@ -85,7 +86,7 @@ export class CheckDeletedFilesTask { params: { version: VERSION }, }); } catch (e) { - this.logger.error(`Error scheduling task, received error: ${e}`); + this.logger.error(`Error scheduling task, received error: ${e.message}`, e); } }; @@ -104,19 +105,34 @@ export class CheckDeletedFilesTask { throwUnrecoverableError(new Error('Outdated task version')); } + this.logger.info(`[runTask()] started`); + + const endRun = (msg: string = '') => { + this.logger.info(`[runTask()] ended${msg ? ': ' + msg : ''}`); + }; + const [{ elasticsearch }] = await core.getStartServices(); const esClient = elasticsearch.client.asInternalUser; try { const readyFiles = await getFilesByStatus(esClient, this.abortController); - if (!readyFiles.length) return; + + if (!readyFiles.length) { + endRun('no files to process'); + return; + } const { fileIdsByIndex: deletedFileIdsByIndex, allFileIds: allDeletedFileIds } = await fileIdsWithoutChunksByIndex(esClient, this.abortController, readyFiles); - if (!allDeletedFileIds.size) return; + + if (!allDeletedFileIds.size) { + endRun('No files with deleted chunks'); + return; + } this.logger.info(`Attempting to update ${allDeletedFileIds.size} files to DELETED status`); - this.logger.debug(`Attempting to file ids: ${deletedFileIdsByIndex}`); + this.logger.debug(`Attempting to update file ids: ${deletedFileIdsByIndex}`); + const updatedFilesResponses = await updateFilesStatus( esClient, this.abortController, @@ -130,12 +146,16 @@ export class CheckDeletedFilesTask { this.logger.warn(`Failed to update ${failures.length} files to DELETED status`); this.logger.debug(`Failed to update files to DELETED status: ${failures}`); } + + endRun('success'); } catch (err) { if (err instanceof errors.RequestAbortedError) { this.logger.warn(`request aborted due to timeout: ${err}`); + endRun(); return; } this.logger.error(err); + endRun('error'); } }; } diff --git a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts index fe1b22644d8e1..1d7a69186421f 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/actions.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/actions.ts @@ -466,6 +466,7 @@ export interface FileUploadMetadata { transithash: { sha256: string; }; + '@timestamp': string; } export type UploadedFileInfo = Pick< diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/response_actions.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/response_actions.ts index f8c3aad63f4e2..8a5c77c2a61b0 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/response_actions.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/response_actions.ts @@ -209,8 +209,9 @@ export const sendEndpointActionResponse = async ( const fileMeta = await esClient.index({ index: FILE_STORAGE_METADATA_INDEX, id: getFileDownloadId(action, action.agents[0]), - body: fileMetaDoc, + op_type: 'create', refresh: 'wait_for', + body: fileMetaDoc, }); // Index the file content (just one chunk) @@ -224,12 +225,14 @@ export const sendEndpointActionResponse = async ( document: cborx.encode({ bid: fileMeta._id, last: true, + '@timestamp': new Date().toISOString(), data: Buffer.from( 'UEsDBAoACQAAAFZeRFWpAsDLHwAAABMAAAAMABwAYmFkX2ZpbGUudHh0VVQJAANTVjxjU1Y8Y3V4CwABBPUBAAAEFAAAAMOcoyEq/Q4VyG02U9O0LRbGlwP/y5SOCfRKqLz1rsBQSwcIqQLAyx8AAAATAAAAUEsBAh4DCgAJAAAAVl5EVakCwMsfAAAAEwAAAAwAGAAAAAAAAQAAAKSBAAAAAGJhZF9maWxlLnR4dFVUBQADU1Y8Y3V4CwABBPUBAAAEFAAAAFBLBQYAAAAAAQABAFIAAAB1AAAAAAA=', 'base64' ), }), refresh: 'wait_for', + op_type: 'create', }, { headers: { diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts index 434df1b209a19..e38725e5d5e6e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/stack_services.ts @@ -44,7 +44,7 @@ export interface RuntimeServices { interface CreateRuntimeServicesOptions { kibanaUrl: string; elasticsearchUrl: string; - fleetServerUrl: string | undefined; + fleetServerUrl?: string; username: string; password: string; log?: ToolingLog; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/mocks.ts index d1818b9b494e7..a4536f54b92d7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/mocks.ts @@ -214,6 +214,7 @@ export const generateFileMetadataDocumentMock = ( transithash: { sha256: 'a0d6d6a2bb73340d4a0ed32b2a46272a19dd111427770c072918aed7a8565010', }, + '@timestamp': new Date().toISOString(), ...overrides, }; diff --git a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts index 6f548d3d955d0..174d511674ef6 100644 --- a/x-pack/test/fleet_api_integration/apis/agents/uploads.ts +++ b/x-pack/test/fleet_api_integration/apis/agents/uploads.ts @@ -23,6 +23,25 @@ export default function (providerContext: FtrProviderContext) { const ES_INDEX_OPTIONS = { headers: { 'X-elastic-product-origin': 'fleet' } }; + const cleanupFiles = async () => { + await esClient.deleteByQuery({ + index: `${FILE_STORAGE_DATA_AGENT_INDEX},${FILE_STORAGE_METADATA_AGENT_INDEX}`, + refresh: true, + ignore_unavailable: true, + query: { + bool: { + filter: [ + { + ids: { + values: ['file1', 'file1.0'], + }, + }, + ], + }, + }, + }); + }; + describe('fleet_uploads', () => { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); @@ -30,6 +49,7 @@ export default function (providerContext: FtrProviderContext) { before(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); await getService('supertest').post(`/api/fleet/setup`).set('kbn-xsrf', 'xxx').send(); + await cleanupFiles(); await esClient.create({ index: AGENT_ACTIONS_INDEX, @@ -60,34 +80,36 @@ export default function (providerContext: FtrProviderContext) { ES_INDEX_OPTIONS ); - await esClient.update({ + await esClient.index({ index: FILE_STORAGE_METADATA_AGENT_INDEX, id: 'file1', refresh: true, + op_type: 'create', body: { - doc_as_upsert: true, - doc: { - upload_id: 'file1', - action_id: 'action1', - agent_id: 'agent1', - file: { - ChunkSize: 4194304, - extension: 'zip', - hash: {}, - mime_type: 'application/zip', - mode: '0644', - name: 'elastic-agent-diagnostics-2022-10-07T12-00-00Z-00.zip', - path: '/agent/elastic-agent-diagnostics-2022-10-07T12-00-00Z-00.zip', - size: 24917, - Status: 'READY', - type: 'file', - }, + '@timestamp': new Date().toISOString(), + upload_id: 'file1', + action_id: 'action1', + agent_id: 'agent1', + file: { + ChunkSize: 4194304, + extension: 'zip', + hash: {}, + mime_type: 'application/zip', + mode: '0644', + name: 'elastic-agent-diagnostics-2022-10-07T12-00-00Z-00.zip', + path: '/agent/elastic-agent-diagnostics-2022-10-07T12-00-00Z-00.zip', + size: 24917, + Status: 'READY', + type: 'file', }, }, }); }); after(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + await Promise.all([ + esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'), + cleanupFiles(), + ]); }); it('should get agent uploads', async () => { @@ -108,17 +130,16 @@ export default function (providerContext: FtrProviderContext) { }); it('should get agent uploaded file', async () => { - await esClient.update({ + await esClient.index({ index: FILE_STORAGE_DATA_AGENT_INDEX, id: 'file1.0', + op_type: 'create', refresh: true, body: { - doc_as_upsert: true, - doc: { - last: true, - bid: 'file1', - data: 'test', - }, + '@timestamp': new Date().toISOString(), + last: true, + bid: 'file1', + data: 'test', }, }); diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/file_upload_index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/file_upload_index.ts deleted file mode 100644 index ce8179a050c47..0000000000000 --- a/x-pack/test/security_solution_endpoint_api_int/apis/file_upload_index.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { - FILE_STORAGE_DATA_INDEX, - FILE_STORAGE_METADATA_INDEX, -} from '@kbn/security-solution-plugin/common/endpoint/constants'; -import { FtrProviderContext } from '../ftr_provider_context'; - -export default function ({ getService }: FtrProviderContext) { - const esClient = getService('es'); - - describe('File upload indices', () => { - it('should have created the file data index on install', async () => { - const endpointFileUploadIndexExists = await esClient.indices.exists({ - index: FILE_STORAGE_METADATA_INDEX, - }); - - expect(endpointFileUploadIndexExists).equal(true); - }); - it('should have created the files index on install', async () => { - const endpointFileUploadIndexExists = await esClient.indices.exists({ - index: FILE_STORAGE_DATA_INDEX, - }); - - expect(endpointFileUploadIndexExists).equal(true); - }); - }); -} diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/index.ts b/x-pack/test/security_solution_endpoint_api_int/apis/index.ts index fbd33d38d1a94..e68db8182da20 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/index.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/index.ts @@ -48,7 +48,6 @@ export default function endpointAPIIntegrationTests(providerContext: FtrProvider loadTestFile(require.resolve('./package')); loadTestFile(require.resolve('./endpoint_authz')); loadTestFile(require.resolve('./endpoint_response_actions/execute')); - loadTestFile(require.resolve('./file_upload_index')); loadTestFile(require.resolve('./endpoint_artifacts/trusted_apps')); loadTestFile(require.resolve('./endpoint_artifacts/event_filters')); loadTestFile(require.resolve('./endpoint_artifacts/host_isolation_exceptions'));