[Fleet] Update fleet file storage indices to the new Datastream names (…

…#160998) ## Summary - Updates fleet file storage indexes to the new Datastream names: | Old Name | New Name | |-----------|-------------------| | `.fleet-file-data-*` | `.fleet-fileds-fromhost-data-*` | | `.fleet-files-*` | `.fleet-fileds-fromhost-meta-*` | | `.fleet-filedelivery-data-*` | `.fleet-fileds-tohost-data-*` | | `.fleet-filedelivery-meta-*` | `.fleet-fileds-tohost-meta-*` | - Removes code that was initializing the old backing indexes - Updates the `fleet:check-deleted-files-task` to ensure it correctly parses the index name/alias from the underlying chunk backing index - Update Security Solution dev scripts, types and mocks to include the `@timestamp` property and ensure any mocks indexed use `op_type:create`
elastic · Jul 3, 2023 · 66fd6eb · 66fd6eb
1 parent e8b2303
commit 66fd6eb
Show file tree

Hide file tree

Showing 19 changed files with 187 additions and 281 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/file_storage.ts b/x-pack/plugins/fleet/common/constants/file_storage.ts
@@ -8,26 +8,26 @@
 // File storage indexes supporting file upload from the host to Elastic/Kibana
 // If needing to get an integration specific index name, use the utility functions
 // found in `common/services/file_storage`
-export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-files-*';
-export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-file-data-*';
+export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-fileds-fromhost-meta-*';
+export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-fileds-fromhost-data-*';
 
-// File storage indexes supporting user uplaoded files (via kibana) that will be
+// File storage indexes supporting user uploaded files (via kibana) that will be
 // delivered to the host agent/endpoint
-export const FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN = '.fleet-filedelivery-meta-*';
-export const FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN = '.fleet-filedelivery-data-*';
+export const FILE_STORAGE_TO_HOST_METADATA_INDEX_PATTERN = '.fleet-fileds-tohost-meta-*';
+export const FILE_STORAGE_TO_HOST_DATA_INDEX_PATTERN = '.fleet-fileds-tohost-data-*';
 
 // which integrations support file upload and the name to use for the file upload index
 export const FILE_STORAGE_INTEGRATION_INDEX_NAMES: Readonly<
   Record<
     string,
-    {
+    Readonly<{
       /** name to be used for the index */
       name: string;
       /** If integration supports files sent from host to ES/Kibana */
       fromHost: boolean;
       /** If integration supports files to be sent to host from kibana */
       toHost: boolean;
-    }
+    }>
   >
 > = {
   elastic_agent: { name: 'agent', fromHost: true, toHost: false },

diff --git a/x-pack/plugins/fleet/common/services/file_storage.test.ts b/x-pack/plugins/fleet/common/services/file_storage.test.ts
@@ -5,24 +5,41 @@
  * 2.0.
  */
 
+import { FILE_STORAGE_METADATA_INDEX_PATTERN } from '../constants';
+
 import { getFileDataIndexName, getFileMetadataIndexName } from '..';
 
+import { getIntegrationNameFromIndexName } from './file_storage';
+
 describe('File Storage services', () => {
   describe('File Index Names', () => {
     it('should generate file metadata index name for files received from host', () => {
-      expect(getFileMetadataIndexName('foo')).toEqual('.fleet-files-foo');
+      expect(getFileMetadataIndexName('foo')).toEqual('.fleet-fileds-fromhost-meta-foo');
     });
 
     it('should generate file data index name for files received from host', () => {
-      expect(getFileDataIndexName('foo')).toEqual('.fleet-file-data-foo');
+      expect(getFileDataIndexName('foo')).toEqual('.fleet-fileds-fromhost-data-foo');
     });
 
     it('should generate file metadata index name for files to be delivered to host', () => {
-      expect(getFileMetadataIndexName('foo', true)).toEqual('.fleet-filedelivery-meta-foo');
+      expect(getFileMetadataIndexName('foo', true)).toEqual('.fleet-fileds-tohost-meta-foo');
     });
 
     it('should generate file data index name for files to be delivered to host', () => {
-      expect(getFileDataIndexName('foo', true)).toEqual('.fleet-filedelivery-data-foo');
+      expect(getFileDataIndexName('foo', true)).toEqual('.fleet-fileds-tohost-data-foo');
+    });
+  });
+
+  describe('getIntegrationNameFromIndexName()', () => {
+    it.each([
+      ['regular index names', '.fleet-fileds-fromhost-meta-agent'],
+      ['datastream index names', '.ds-.fleet-fileds-fromhost-data-agent-2023.06.30-00001'],
+    ])('should handle %s', (_, index) => {
+      expect(getIntegrationNameFromIndexName(index, FILE_STORAGE_METADATA_INDEX_PATTERN)).toEqual(
+        'agent'
+      );
     });
+
+    it.todo('should error if index pattern does not include `*`');
   });
 });
diff --git a/x-pack/plugins/fleet/common/services/file_storage.ts b/x-pack/plugins/fleet/common/services/file_storage.ts
@@ -56,21 +56,19 @@ export const getFileDataIndexName = (
   );
 };
 
-/**
- * Returns the write index name for a given file upload alias name, this is the same for metadata and chunks
- * @param aliasName
- */
-export const getFileWriteIndexName = (aliasName: string) => aliasName + '-000001';
 /**
  * Returns back the integration name for a given File Data (chunks) index name.
  *
  * @example
- * // Given a File data index pattern of `.fleet-file-data-*`:
+ * // Given a File data index pattern of `.fleet-fileds-fromhost-data-*`:
  *
- * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent');
+ * getIntegrationNameFromFileDataIndexName('.fleet-fileds-fromhost-data-agent');
  * // return 'agent'
  *
- * getIntegrationNameFromFileDataIndexName('.fleet-file-data-agent-00001');
+ * getIntegrationNameFromFileDataIndexName('.ds-.fleet-fileds-fromhost-data-agent');
+ * // return 'agent'
+ *
+ * getIntegrationNameFromFileDataIndexName('.ds-.fleet-fileds-fromhost-data-agent-2023.06.30-00001');
  * // return 'agent'
  */
 export const getIntegrationNameFromFileDataIndexName = (indexName: string): string => {
@@ -87,23 +85,11 @@ export const getIntegrationNameFromIndexName = (
     throw new Error(`Unable to parse index name. No '*' in index pattern: ${indexPattern}`);
   }
 
-  const indexPieces = indexName.split('-');
+  const indexPieces = indexName.replace(/^\.ds-/, '').split('-');
 
   if (indexPieces[integrationNameIndexPosition]) {
     return indexPieces[integrationNameIndexPosition];
   }
 
   throw new Error(`Index name ${indexName} does not seem to be a File storage index`);
 };
-
-export const getFileStorageWriteIndexBody = (aliasName: string) => ({
-  aliases: {
-    [aliasName]: {
-      is_write_index: true,
-    },
-  },
-  settings: {
-    'index.lifecycle.rollover_alias': aliasName,
-    'index.hidden': true,
-  },
-});
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
@@ -11,18 +11,9 @@ import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 
 import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types';
 
-import {
-  FILE_STORAGE_INTEGRATION_INDEX_NAMES,
-  FILE_STORAGE_INTEGRATION_NAMES,
-} from '../../../../../common/constants';
-
 import { ElasticsearchAssetType } from '../../../../types';
 import {
-  getFileWriteIndexName,
-  getFileStorageWriteIndexBody,
   getPipelineNameForDatastream,
-  getFileDataIndexName,
-  getFileMetadataIndexName,
   getRegistryDataStreamAssetBaseName,
 } from '../../../../../common/services';
 import type {
@@ -440,63 +431,6 @@ export async function ensureDefaultComponentTemplates(
   );
 }
 
-/*
- * Given a list of integration names, if the integrations support file upload
- * then ensure that the alias has a matching write index, as we use "plain" indices
- * not data streams.
- * e.g .fleet-file-data-agent must have .fleet-file-data-agent-00001 as the write index
- * before files can be uploaded.
- */
-export async function ensureFileUploadWriteIndices(opts: {
-  esClient: ElasticsearchClient;
-  logger: Logger;
-  integrationNames: string[];
-}) {
-  const { esClient, logger, integrationNames } = opts;
-
-  const integrationsWithFileUpload = integrationNames.filter((integration) =>
-    FILE_STORAGE_INTEGRATION_NAMES.includes(integration as any)
-  );
-
-  if (!integrationsWithFileUpload.length) return [];
-
-  const ensure = (aliasName: string) =>
-    ensureAliasHasWriteIndex({
-      esClient,
-      logger,
-      aliasName,
-      writeIndexName: getFileWriteIndexName(aliasName),
-      body: getFileStorageWriteIndexBody(aliasName),
-    });
-
-  return Promise.all(
-    integrationsWithFileUpload.flatMap((integrationName) => {
-      const {
-        name: indexName,
-        fromHost,
-        toHost,
-      } = FILE_STORAGE_INTEGRATION_INDEX_NAMES[integrationName];
-      const indexCreateRequests: Array<Promise<void>> = [];
-
-      if (fromHost) {
-        indexCreateRequests.push(
-          ensure(getFileDataIndexName(indexName)),
-          ensure(getFileMetadataIndexName(indexName))
-        );
-      }
-
-      if (toHost) {
-        indexCreateRequests.push(
-          ensure(getFileDataIndexName(indexName, true)),
-          ensure(getFileMetadataIndexName(indexName, true))
-        );
-      }
-
-      return indexCreateRequests;
-    })
-  );
-}
-
 export async function ensureComponentTemplate(
   esClient: ElasticsearchClient,
   logger: Logger,

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
@@ -37,7 +37,6 @@ import type {
   PackageVerificationResult,
   IndexTemplateEntry,
 } from '../../../types';
-import { ensureFileUploadWriteIndices } from '../elasticsearch/template/install';
 import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy';
 import { isTopLevelPipeline, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline';
 import { installILMPolicy } from '../elasticsearch/ilm/install';
@@ -236,15 +235,6 @@ export async function _installPackage({
       logger.warn(`Error removing legacy templates: ${e.message}`);
     }
 
-    const { diagnosticFileUploadEnabled } = appContextService.getExperimentalFeatures();
-    if (diagnosticFileUploadEnabled) {
-      await ensureFileUploadWriteIndices({
-        integrationNames: [packageInfo.name],
-        esClient,
-        logger,
-      });
-    }
-
     // update current backing indices of each data stream
     await withPackageSpan('Update write indices', () =>
       updateCurrentWriteIndices(esClient, logger, indexTemplates)

diff --git a/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts b/x-pack/plugins/fleet/server/services/files/client_from_host.test.ts
@@ -91,11 +91,11 @@ describe('FleetFromHostFilesClient', () => {
 
     esClientMock.search.mockImplementation(async (searchRequest = {}) => {
       // File metadata
-      if ((searchRequest.index as string).startsWith('.fleet-files-')) {
+      if ((searchRequest.index as string).startsWith('.fleet-fileds-fromhost-meta-')) {
         return fleetFilesIndexSearchResponse;
       }
 
-      if ((searchRequest.index as string).startsWith('.fleet-file-data-')) {
+      if ((searchRequest.index as string).startsWith('.fleet-fileds-fromhost-data-')) {
         return fleetFileDataIndexSearchResponse;
       }
 
@@ -111,8 +111,8 @@ describe('FleetFromHostFilesClient', () => {
     expect(createEsFileClientMock).toHaveBeenCalledWith({
       elasticsearchClient: esClientMock,
       logger: loggerMock,
-      metadataIndex: '.fleet-files-foo',
-      blobStorageIndex: '.fleet-file-data-foo',
+      metadataIndex: '.fleet-fileds-fromhost-meta-foo',
+      blobStorageIndex: '.fleet-fileds-fromhost-data-foo',
       indexIsAlias: true,
     });
   });
@@ -159,7 +159,7 @@ describe('FleetFromHostFilesClient', () => {
             },
           },
         },
-        index: '.fleet-file-data-foo',
+        index: '.fleet-fileds-fromhost-data-foo',
         size: 0,
       });
     });

diff --git a/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts b/x-pack/plugins/fleet/server/services/files/client_to_host.test.ts
@@ -130,8 +130,8 @@ describe('FleetToHostFilesClient', () => {
     expect(createEsFileClientMock).toHaveBeenCalledWith({
       elasticsearchClient: esClientMock,
       logger: loggerMock,
-      metadataIndex: '.fleet-filedelivery-meta-foo',
-      blobStorageIndex: '.fleet-filedelivery-data-foo',
+      metadataIndex: '.fleet-fileds-tohost-meta-foo',
+      blobStorageIndex: '.fleet-fileds-tohost-data-foo',
       maxSizeBytes: 12345,
       indexIsAlias: true,
     });