[Fleet] Allow snake cased Kibana assets (elastic#77515)

* Properly handle kibana assets with underscores in their path

* Recomment test

* Fix type check

* Don't install index patterns that are reserved

* Introduce SavedObjectType to use on AssetReference

* Fix Test

* Update install.ts

Use new `dataTypes` const which replaced `DataType` enum

* Update install.ts

Remove unused `indexPatternTypes` from outer scope

* Update install.ts

fix (?) bad updates from before where new/correct value was used but result wasn't exported

* Update install.ts

* Update install.ts

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>
Co-authored-by: John Schulz <[email protected]>

x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts

@@ -25,7 +25,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
         dashboard: [],
         visualization: [],
         search: [],
-        'index-pattern': [],
+        index_pattern: [],
         map: [],
       },
     },

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -35,7 +35,21 @@ export type ServiceName = 'kibana' | 'elasticsearch';
 export type AgentAssetType = typeof agentAssetTypes;
 export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf<AgentAssetType>;
 
+/*
+  Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased)
+*/
 export enum KibanaAssetType {
+  dashboard = 'dashboard',
+  visualization = 'visualization',
+  search = 'search',
+  indexPattern = 'index_pattern',
+  map = 'map',
+}
+
+/*
+ Enum of saved object types that are allowed to be installed 
+*/
+export enum KibanaSavedObjectType {
   dashboard = 'dashboard',
   visualization = 'visualization',
   search = 'search',
@@ -271,7 +285,7 @@ export type NotInstalled<T = {}> = T & {
 export type AssetReference = KibanaAssetReference | EsAssetReference;
 
 export type KibanaAssetReference = Pick<SavedObjectReference, 'id'> & {
-  type: KibanaAssetType;
+  type: KibanaSavedObjectType;
 };
 export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: ElasticsearchAssetType;

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx

@@ -20,7 +20,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
   ilm_policy: 'ILM Policy',
   ingest_pipeline: 'Ingest Pipeline',
   transform: 'Transform',
-  'index-pattern': 'Index Pattern',
+  index_pattern: 'Index Pattern',
   index_template: 'Index Template',
   component_template: 'Component Template',
   search: 'Saved Search',
@@ -36,7 +36,7 @@ export const ServiceTitleMap: Record<ServiceName, string> = {
 
 export const AssetIcons: Record<KibanaAssetType, IconType> = {
   dashboard: 'dashboardApp',
-  'index-pattern': 'indexPatternApp',
+  index_pattern: 'indexPatternApp',
   search: 'searchProfilerApp',
   visualization: 'visualizeApp',
   map: 'mapApp',

x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts

@@ -5,7 +5,7 @@
  */
 import { RequestHandler, SavedObjectsClientContract } from 'src/core/server';
 import { DataStream } from '../../types';
-import { GetDataStreamsResponse, KibanaAssetType } from '../../../common';
+import { GetDataStreamsResponse, KibanaAssetType, KibanaSavedObjectType } from '../../../common';
 import { getPackageSavedObjects, getKibanaSavedObject } from '../../services/epm/packages/get';
 import { defaultIngestErrorHandler } from '../../errors';
 
@@ -124,7 +124,7 @@ export const getListHandler: RequestHandler = async (context, request, response)
         // then pick the dashboards from the package saved object
         const dashboards =
           pkgSavedObject[0].attributes?.installed_kibana?.filter(
-            (o) => o.type === KibanaAssetType.dashboard
+            (o) => o.type === KibanaSavedObjectType.dashboard
           ) || [];
         // and then pick the human-readable titles from the dashboard saved objects
         const enhancedDashboards = await getEnhancedDashboards(

x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts

@@ -11,17 +11,49 @@ import {
 } from 'src/core/server';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common';
 import * as Registry from '../../registry';
-import { AssetType, KibanaAssetType, AssetReference } from '../../../../types';
+import {
+  AssetType,
+  KibanaAssetType,
+  AssetReference,
+  AssetParts,
+  KibanaSavedObjectType,
+} from '../../../../types';
 import { savedObjectTypes } from '../../packages';
+import { indexPatternTypes } from '../index_pattern/install';
 
 type SavedObjectToBe = Required<Pick<SavedObjectsBulkCreateObject, keyof ArchiveAsset>> & {
-  type: AssetType;
+  type: KibanaSavedObjectType;
 };
 export type ArchiveAsset = Pick<
   SavedObject,
   'id' | 'attributes' | 'migrationVersion' | 'references'
 > & {
-  type: AssetType;
+  type: KibanaSavedObjectType;
+};
+
+// KibanaSavedObjectTypes are used to ensure saved objects being created for a given
+// KibanaAssetType have the correct type
+const KibanaSavedObjectTypeMapping: Record<KibanaAssetType, KibanaSavedObjectType> = {
+  [KibanaAssetType.dashboard]: KibanaSavedObjectType.dashboard,
+  [KibanaAssetType.indexPattern]: KibanaSavedObjectType.indexPattern,
+  [KibanaAssetType.map]: KibanaSavedObjectType.map,
+  [KibanaAssetType.search]: KibanaSavedObjectType.search,
+  [KibanaAssetType.visualization]: KibanaSavedObjectType.visualization,
+};
+
+// Define how each asset type will be installed
+const AssetInstallers: Record<
+  KibanaAssetType,
+  (args: {
+    savedObjectsClient: SavedObjectsClientContract;
+    kibanaAssets: ArchiveAsset[];
+  }) => Promise<Array<SavedObject<unknown>>>
+> = {
+  [KibanaAssetType.dashboard]: installKibanaSavedObjects,
+  [KibanaAssetType.indexPattern]: installKibanaIndexPatterns,
+  [KibanaAssetType.map]: installKibanaSavedObjects,
+  [KibanaAssetType.search]: installKibanaSavedObjects,
+  [KibanaAssetType.visualization]: installKibanaSavedObjects,
 };
 
 export async function getKibanaAsset(key: string): Promise<ArchiveAsset> {
@@ -47,16 +79,22 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo
 export async function installKibanaAssets(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
-  kibanaAssets: ArchiveAsset[];
+  kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
 }): Promise<SavedObject[]> {
   const { savedObjectsClient, kibanaAssets } = options;
 
   // install the assets
   const kibanaAssetTypes = Object.values(KibanaAssetType);
   const installedAssets = await Promise.all(
-    kibanaAssetTypes.map((assetType) =>
-      installKibanaSavedObjects({ savedObjectsClient, assetType, kibanaAssets })
-    )
+    kibanaAssetTypes.map((assetType) => {
+      if (kibanaAssets[assetType]) {
+        return AssetInstallers[assetType]({
+          savedObjectsClient,
+          kibanaAssets: kibanaAssets[assetType],
+        });
+      }
+      return [];
+    })
   );
   return installedAssets.flat();
 }
@@ -74,25 +112,50 @@ export const deleteKibanaInstalledRefs = async (
     installed_kibana: installedAssetsToSave,
   });
 };
-export async function getKibanaAssets(paths: string[]) {
-  const isKibanaAssetType = (path: string) => Registry.pathParts(path).type in KibanaAssetType;
-  const filteredPaths = paths.filter(isKibanaAssetType);
-  const kibanaAssets = await Promise.all(filteredPaths.map((path) => getKibanaAsset(path)));
-  return kibanaAssets;
+export async function getKibanaAssets(
+  paths: string[]
+): Promise<Record<KibanaAssetType, ArchiveAsset[]>> {
+  const kibanaAssetTypes = Object.values(KibanaAssetType);
+  const isKibanaAssetType = (path: string) => {
+    const parts = Registry.pathParts(path);
+
+    return parts.service === 'kibana' && (kibanaAssetTypes as string[]).includes(parts.type);
+  };
+
+  const filteredPaths = paths
+    .filter(isKibanaAssetType)
+    .map<[string, AssetParts]>((path) => [path, Registry.pathParts(path)]);
+
+  const assetArrays: Array<Promise<ArchiveAsset[]>> = [];
+  for (const assetType of kibanaAssetTypes) {
+    const matching = filteredPaths.filter(([path, parts]) => parts.type === assetType);
+
+    assetArrays.push(Promise.all(matching.map(([path]) => path).map(getKibanaAsset)));
+  }
+
+  const resolvedAssets = await Promise.all(assetArrays);
+
+  const result = {} as Record<KibanaAssetType, ArchiveAsset[]>;
+
+  for (const [index, assetType] of kibanaAssetTypes.entries()) {
+    const expectedType = KibanaSavedObjectTypeMapping[assetType];
+    const properlyTypedAssets = resolvedAssets[index].filter(({ type }) => type === expectedType);
+
+    result[assetType] = properlyTypedAssets;
+  }
+
+  return result;
 }
+
 async function installKibanaSavedObjects({
   savedObjectsClient,
-  assetType,
   kibanaAssets,
 }: {
   savedObjectsClient: SavedObjectsClientContract;
-  assetType: KibanaAssetType;
   kibanaAssets: ArchiveAsset[];
 }) {
-  const isSameType = (asset: ArchiveAsset) => assetType === asset.type;
-  const filteredKibanaAssets = kibanaAssets.filter((asset) => isSameType(asset));
   const toBeSavedObjects = await Promise.all(
-    filteredKibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset))
+    kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset))
   );
 
   if (toBeSavedObjects.length === 0) {
@@ -105,8 +168,23 @@ async function installKibanaSavedObjects({
   }
 }
 
+async function installKibanaIndexPatterns({
+  savedObjectsClient,
+  kibanaAssets,
+}: {
+  savedObjectsClient: SavedObjectsClientContract;
+  kibanaAssets: ArchiveAsset[];
+}) {
+  // Filter out any reserved index patterns
+  const reservedPatterns = indexPatternTypes.map((pattern) => `${pattern}-*`);
+
+  const nonReservedPatterns = kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id));
+
+  return installKibanaSavedObjects({ savedObjectsClient, kibanaAssets: nonReservedPatterns });
+}
+
 export function toAssetReference({ id, type }: SavedObject) {
-  const reference: AssetReference = { id, type: type as KibanaAssetType };
+  const reference: AssetReference = { id, type: type as KibanaSavedObjectType };
 
   return reference;
 }

x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts

@@ -72,6 +72,7 @@ export interface IndexPatternField {
   readFromDocValues: boolean;
 }
 
+export const indexPatternTypes = Object.values(dataTypes);
 // TODO: use a function overload and make pkgName and pkgVersion required for install/update
 // and not for an update removal.  or separate out the functions
 export async function installIndexPatterns(
@@ -116,7 +117,6 @@ export async function installIndexPatterns(
   const packageVersionsInfo = await Promise.all(packageVersionsFetchInfoPromise);
 
   // for each index pattern type, create an index pattern
-  const indexPatternTypes = Object.values(dataTypes);
   indexPatternTypes.forEach(async (indexPatternType) => {
     // if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
     if (!pkgName && installedPackages.length === 0) {

x-pack/plugins/ingest_manager/server/services/epm/packages/ensure_installed_default_packages.test.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types';
+import { ElasticsearchAssetType, Installation, KibanaSavedObjectType } from '../../../types';
 import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
 
 jest.mock('./install');
@@ -41,7 +41,7 @@ const mockInstallation: SavedObject<Installation> = {
   type: 'epm-packages',
   attributes: {
     id: 'test-pkg',
-    installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
+    installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test package',

x-pack/plugins/ingest_manager/server/services/epm/packages/get_install_type.test.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { SavedObject } from 'src/core/server';
-import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types';
+import { ElasticsearchAssetType, Installation, KibanaSavedObjectType } from '../../../types';
 import { getInstallType } from './install';
 
 const mockInstallation: SavedObject<Installation> = {
@@ -13,7 +13,7 @@ const mockInstallation: SavedObject<Installation> = {
   type: 'epm-packages',
   attributes: {
     id: 'test-pkg',
-    installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
+    installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',
@@ -30,7 +30,7 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
   type: 'epm-packages',
   attributes: {
     id: 'test-pkg',
-    installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
+    installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
     installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
     es_index_patterns: { pattern: 'pattern-name' },
     name: 'test packagek',

x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts

@@ -18,6 +18,7 @@ import {
   KibanaAssetReference,
   EsAssetReference,
   InstallType,
+  KibanaAssetType,
 } from '../../../types';
 import * as Registry from '../registry';
 import {
@@ -364,9 +365,9 @@ export async function createInstallation(options: {
 export const saveKibanaAssetsRefs = async (
   savedObjectsClient: SavedObjectsClientContract,
   pkgName: string,
-  kibanaAssets: ArchiveAsset[]
+  kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>
 ) => {
-  const assetRefs = kibanaAssets.map(toAssetReference);
+  const assetRefs = Object.values(kibanaAssets).flat().map(toAssetReference);
   await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
     installed_kibana: assetRefs,
   });

x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts

@@ -12,6 +12,9 @@ import {
   AssetType,
   CallESAsCurrentUser,
   ElasticsearchAssetType,
+  EsAssetReference,
+  KibanaAssetReference,
+  Installation,
 } from '../../../types';
 import { getInstallation, savedObjectTypes } from './index';
 import { deletePipeline } from '../elasticsearch/ingest_pipeline/';
@@ -46,7 +49,7 @@ export async function removeInstallation(options: {
 
   // Delete the installed assets
   const installedAssets = [...installation.installed_kibana, ...installation.installed_es];
-  await deleteAssets(installedAssets, savedObjectsClient, callCluster);
+  await deleteAssets(installation, savedObjectsClient, callCluster);
 
   // Delete the manager saved object with references to the asset objects
   // could also update with [] or some other state
@@ -64,26 +67,43 @@ export async function removeInstallation(options: {
   // successful delete's in SO client return {}. return something more useful
   return installedAssets;
 }
-async function deleteAssets(
-  installedObjects: AssetReference[],
-  savedObjectsClient: SavedObjectsClientContract,
-  callCluster: CallESAsCurrentUser
+
+function deleteKibanaAssets(
+  installedObjects: KibanaAssetReference[],
+  savedObjectsClient: SavedObjectsClientContract
 ) {
-  const logger = appContextService.getLogger();
-  const deletePromises = installedObjects.map(async ({ id, type }) => {
+  return installedObjects.map(async ({ id, type }) => {
+    return savedObjectsClient.delete(type, id);
+  });
+}
+
+function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallESAsCurrentUser) {
+  return installedObjects.map(async ({ id, type }) => {
     const assetType = type as AssetType;
-    if (savedObjectTypes.includes(assetType)) {
-      return savedObjectsClient.delete(assetType, id);
-    } else if (assetType === ElasticsearchAssetType.ingestPipeline) {
+    if (assetType === ElasticsearchAssetType.ingestPipeline) {
       return deletePipeline(callCluster, id);
     } else if (assetType === ElasticsearchAssetType.indexTemplate) {
       return deleteTemplate(callCluster, id);
     } else if (assetType === ElasticsearchAssetType.transform) {
       return deleteTransforms(callCluster, [id]);
     }
   });
+}
+
+async function deleteAssets(
+  { installed_es: installedEs, installed_kibana: installedKibana }: Installation,
+  savedObjectsClient: SavedObjectsClientContract,
+  callCluster: CallESAsCurrentUser
+) {
+  const logger = appContextService.getLogger();
+
+  const deletePromises: Array<Promise<unknown>> = [
+    ...deleteESAssets(installedEs, callCluster),
+    ...deleteKibanaAssets(installedKibana, savedObjectsClient),
+  ];
+
   try {
-    await Promise.all([...deletePromises]);
+    await Promise.all(deletePromises);
   } catch (err) {
     logger.error(err);
   }