[Fleet] Use default component templates from Elasticsearch

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -615,6 +615,7 @@ export interface IndexTemplate {
   };
   data_stream: { hidden?: boolean };
   composed_of: string[];
+  ignore_missing_component_templates?: string[];
   _meta: object;
 }
 

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -11,7 +11,7 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';
 
 const meta = getESAssetMetadata();
 
-export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0';
+export const FLEET_INSTALL_FORMAT_VERSION = '1.1.0';
 
 export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1';
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts

@@ -65,13 +65,9 @@ describe('buildDefaultSettings', () => {
     expect(settings).toMatchInlineSnapshot(`
       Object {
         "index": Object {
-          "codec": "best_compression",
           "lifecycle": Object {
             "name": "logs",
           },
-          "mapping": Object {
-            "ignore_malformed": true,
-          },
           "query": Object {
             "default_field": Array [
               "field1Keyword",

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts

@@ -73,16 +73,6 @@ export function buildDefaultSettings({
               name: ilmPolicy ? ilmPolicy : type,
             },
           }),
-      // What should be our default for the compression?
-      codec: 'best_compression',
-      // setting `ignore_malformed` only for data_stream for logs
-      ...(type === 'logs'
-        ? {
-            mapping: {
-              ignore_malformed: true,
-            },
-          }
-        : {}),
       // All the default fields which should be queried have to be added here.
       // So far we add all keyword and text fields here if there are any, otherwise
       // this setting is skipped.

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts

@@ -7,9 +7,9 @@
 import { createAppContextStartContractMock } from '../../../../mocks';
 import { appContextService } from '../../..';
 import { loadFieldsFromYaml } from '../../fields/field';
-import type { RegistryDataStream } from '../../../../types';
+import type { ArchivePackage, RegistryDataStream } from '../../../../types';
 
-import { prepareTemplate } from './install';
+import { prepareTemplate, prepareToInstallTemplates } from './install';
 
 jest.mock('../../fields/field', () => ({
   ...jest.requireActual('../../fields/field'),
@@ -455,4 +455,24 @@ describe('EPM index template install', () => {
 
     expect(packageTemplate).not.toHaveProperty('lifecycle');
   });
+
+  test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => {
+    const dataStreamDatasetIsPrefixUnset = {
+      type: 'logs',
+      dataset: 'package.dataset',
+      title: 'test data stream',
+      release: 'experimental',
+      package: 'package',
+      path: 'path',
+      ingest_pipeline: 'default',
+    } as RegistryDataStream;
+
+    const { assetsToAdd } = prepareToInstallTemplates({
+      name: 'package',
+      version: '0.0.1',
+      data_streams: [dataStreamDatasetIsPrefixUnset],
+    } as ArchivePackage, [], []);
+
+    expect(assetsToAdd).not.toContainEqual({ id: 'logs@settings', type: 'component_template' });
+  })
 });

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -530,7 +530,7 @@ export function prepareTemplate({
 
   const isIndexModeTimeSeries =
     dataStream.elasticsearch?.index_mode === 'time_series' ||
-    experimentalDataStreamFeature?.features.tsdb;
+    !!experimentalDataStreamFeature?.features.tsdb;
 
   const validFields = processFields(fields);
 
@@ -572,6 +572,7 @@ export function prepareTemplate({
     registryElasticsearch: dataStream.elasticsearch,
     mappings,
     isIndexModeTimeSeries,
+    type: dataStream.type,
   });
 
   return {
@@ -616,6 +617,14 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
       .filter(
         (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId)
       )
+      // Filter stack component templates shared between integrations
+      .filter(
+        (componentTemplateId) => [
+          `metrics@tsdb-settings`,
+          `logs@settings`,
+          `metrics@settings`
+        ].indexOf(componentTemplateId) === -1
+      )
       .map((componentTemplateId) => ({
         id: componentTemplateId,
         type: ElasticsearchAssetType.componentTemplate,

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -57,10 +57,13 @@ describe('EPM template', () => {
 
     const template = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(template.index_patterns).toStrictEqual([templateIndexPattern]);
   });
@@ -69,18 +72,58 @@ describe('EPM template', () => {
     const composedOfTemplates = ['component1', 'component2'];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(template.composed_of).toStrictEqual([
+      'logs@settings',
       ...composedOfTemplates,
       ...FLEET_COMPONENT_TEMPLATES_NAMES,
     ]);
   });
 
+  it('supplies metrics@tsdb-settings for time series', () => {
+    const composedOfTemplates = ['component1', 'component2'];
+
+    const template = getTemplate({
+      templateIndexPattern: 'metrics-*',
+      type: 'metrics',
+      packageName: 'nginx',
+      composedOfTemplates,
+      templatePriority: 200,
+      mappings: { properties: [] },
+      isIndexModeTimeSeries: true,
+      isILMPolicyDisabled: false,
+    });
+    expect(template.composed_of).toStrictEqual([
+      'metrics@tsdb-settings',
+      ...composedOfTemplates,
+      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+    ]);
+  });
+
+  it('sets lifecycle policy to null when ILM is disabled', () => {
+    const composedOfTemplates = ['component1', 'component2'];
+
+    const template = getTemplate({
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
+      packageName: 'nginx',
+      composedOfTemplates,
+      templatePriority: 200,
+      mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: true,
+    });
+    expect(template.template.settings.index.lifecycle).toBeNull();
+  });
+
   it('does not create fleet agent id verification component template if agentIdVerification is disabled', () => {
     appContextService.start(
       createAppContextStartContractMock({
@@ -90,13 +133,17 @@ describe('EPM template', () => {
     const composedOfTemplates = ['component1', 'component2'];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(template.composed_of).toStrictEqual([
+      'logs@settings',
       ...composedOfTemplates,
       FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
     ]);
@@ -106,34 +153,46 @@ describe('EPM template', () => {
     const composedOfTemplates: string[] = [];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
-    expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES_NAMES);
+    expect(template.composed_of).toStrictEqual([
+      'logs@settings',
+      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+    ]);
   });
 
   it('adds hidden field correctly', () => {
     const templateIndexPattern = 'logs-nginx.access-abcd-*';
 
     const templateWithHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       hidden: true,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithHidden.data_stream.hidden).toEqual(true);
 
     const templateWithoutHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
   });
@@ -143,6 +202,7 @@ describe('EPM template', () => {
 
     const templateWithGlobalAndDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -153,11 +213,14 @@ describe('EPM template', () => {
           hidden: true,
         },
       },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithGlobalAndDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -167,21 +230,27 @@ describe('EPM template', () => {
           hidden: true,
         },
       },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithoutDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       hidden: true,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithoutDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithGlobalHiddenTrueAndDataStreamHiddenFalse = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -192,15 +261,20 @@ describe('EPM template', () => {
           hidden: false,
         },
       },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithGlobalHiddenTrueAndDataStreamHiddenFalse.data_stream.hidden).toEqual(true);
 
     const templateWithoutHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
+      isILMPolicyDisabled: false,
     });
     expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
   });

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -76,12 +76,14 @@ export function getTemplate({
   registryElasticsearch,
   mappings,
   isIndexModeTimeSeries,
+  type,
 }: {
   templateIndexPattern: string;
   packageName: string;
   composedOfTemplates: string[];
   templatePriority: number;
   mappings: IndexTemplateMappings;
+  type: string;
   hidden?: boolean;
   registryElasticsearch?: RegistryElasticsearch | undefined;
   isIndexModeTimeSeries?: boolean;
@@ -100,7 +102,10 @@ export function getTemplate({
     throw new Error(`Error template for ${templateIndexPattern} contains a final_pipeline`);
   }
 
+  const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries);
+
   template.composed_of = [
+    ...esBaseComponents,
     ...(template.composed_of || []),
     FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
     ...(appContextService.getConfig()?.agentIdVerificationEnabled
@@ -111,6 +116,18 @@ export function getTemplate({
   return template;
 }
 
+const getBaseEsComponents = (type: string, isIndexModeTimeSeries: boolean): string[] => {
+  if (type === 'traces') {
+    return [];
+  }
+
+  if (type === 'metrics' && isIndexModeTimeSeries) {
+    return [`metrics@tsdb-settings`];
+  }
+
+  return [`${type}@settings`];
+};
+
 /**
  * Generate mapping takes the given nested fields array and creates the Elasticsearch
  * mapping properties out of it.