[Fleet] Use default component templates from Elasticsearch (elastic#1…

…63731) ## Summary Fixes elastic#163141 Fixes elastic#160288 Blocked by: - elastic/elasticsearch#98535 This switches where integrations installed by EPM get their default index settings from to use the [source-of-truth component templates supplied by Elasticsearch](https://github.com/elastic/elasticsearch/tree/main/x-pack/plugin/core/template-resources/src/main/resources). This will help ensure that data streams configured by EPM always get the same defaults as data streams the user creates using the default `logs-*-*` and `metrics-*-*` templates. For now, no default mappings are sourced from Elasticsearch. As part of this change the template format version was incremented to force EPM to reinstall all templates and rollover data streams on the Stack upgrade to the version including this change. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]>
delanni · Nov 6, 2023 · 562a6d8 · 562a6d8
1 parent 89fac93
commit 562a6d8
Show file tree

Hide file tree

Showing 14 changed files with 131 additions and 30 deletions.
diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts
@@ -7,6 +7,7 @@
 import { errors } from '@elastic/elasticsearch';
 import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS } from '@kbn/fleet-plugin/server/constants';
 import {
   BENCHMARK_SCORE_INDEX_DEFAULT_NS,
   BENCHMARK_SCORE_INDEX_PATTERN,
@@ -24,6 +25,10 @@ import { CloudSecurityPostureConfig } from '../config';
 interface IndexTemplateSettings {
   index: {
     default_pipeline: string;
+    codec?: string;
+    mapping?: {
+      ignore_malformed: boolean;
+    };
   };
   lifecycle?: { name: string };
 }
@@ -226,6 +231,10 @@ const updateIndexTemplate = async (
     ...template?.settings, // nothing inside
     index: {
       default_pipeline: latestFindingsPipelineIngestConfig.id,
+      codec: 'best_compression',
+      mapping: {
+        ignore_malformed: true,
+      },
     },
     lifecycle: { name: '' },
   };
@@ -242,7 +251,7 @@ const updateIndexTemplate = async (
         aliases: template?.aliases,
       },
       _meta,
-      composed_of: composedOf,
+      composed_of: composedOf.filter((ct) => ct !== STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS),
     });
 
     logger.info(`Updated index template successfully [Name: ${indexTemplateName}]`);

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -615,6 +615,7 @@ export interface IndexTemplate {
   };
   data_stream: { hidden?: boolean };
   composed_of: string[];
+  ignore_missing_component_templates?: string[];
   _meta: object;
 }
 

diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts
@@ -11,7 +11,7 @@ import { getESAssetMetadata } from '../services/epm/elasticsearch/meta';
 
 const meta = getESAssetMetadata();
 
-export const FLEET_INSTALL_FORMAT_VERSION = '1.0.0';
+export const FLEET_INSTALL_FORMAT_VERSION = '1.1.0';
 
 export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1';
 
@@ -81,6 +81,16 @@ export const FLEET_COMPONENT_TEMPLATES = [
   },
 ];
 
+export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`;
+export const STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS = `metrics@settings`;
+export const STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS = `metrics@tsdb-settings`;
+
+export const STACK_COMPONENT_TEMPLATES = [
+  STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
+  STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS,
+  STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS,
+];
+
 export const FLEET_FINAL_PIPELINE_VERSION = 4;
 
 // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline

diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts
@@ -95,6 +95,10 @@ export {
   FLEET_FINAL_PIPELINE_VERSION,
   FLEET_INSTALL_FORMAT_VERSION,
   FLEET_AGENT_POLICIES_SCHEMA_VERSION,
+  STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS,
+  STACK_COMPONENT_TEMPLATE_METRICS_SETTINGS,
+  STACK_COMPONENT_TEMPLATE_METRICS_TSDB_SETTINGS,
+  STACK_COMPONENT_TEMPLATES,
 } from './fleet_es_assets';
 export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets';
 export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets';

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts
@@ -65,13 +65,9 @@ describe('buildDefaultSettings', () => {
     expect(settings).toMatchInlineSnapshot(`
       Object {
         "index": Object {
-          "codec": "best_compression",
           "lifecycle": Object {
             "name": "logs",
           },
-          "mapping": Object {
-            "ignore_malformed": true,
-          },
           "query": Object {
             "default_field": Array [
               "field1Keyword",

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts
@@ -73,16 +73,6 @@ export function buildDefaultSettings({
               name: ilmPolicy ? ilmPolicy : type,
             },
           }),
-      // What should be our default for the compression?
-      codec: 'best_compression',
-      // setting `ignore_malformed` only for data_stream for logs
-      ...(type === 'logs'
-        ? {
-            mapping: {
-              ignore_malformed: true,
-            },
-          }
-        : {}),
       // All the default fields which should be queried have to be added here.
       // So far we add all keyword and text fields here if there are any, otherwise
       // this setting is skipped.

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
@@ -7,9 +7,9 @@
 import { createAppContextStartContractMock } from '../../../../mocks';
 import { appContextService } from '../../..';
 import { loadFieldsFromYaml } from '../../fields/field';
-import type { RegistryDataStream } from '../../../../types';
+import type { ArchivePackage, RegistryDataStream } from '../../../../types';
 
-import { prepareTemplate } from './install';
+import { prepareTemplate, prepareToInstallTemplates } from './install';
 
 jest.mock('../../fields/field', () => ({
   ...jest.requireActual('../../fields/field'),
@@ -455,4 +455,28 @@ describe('EPM index template install', () => {
 
     expect(packageTemplate).not.toHaveProperty('lifecycle');
   });
+
+  test('test prepareToInstallTemplates does not include stack component templates in tracked assets', () => {
+    const dataStreamDatasetIsPrefixUnset = {
+      type: 'logs',
+      dataset: 'package.dataset',
+      title: 'test data stream',
+      release: 'experimental',
+      package: 'package',
+      path: 'path',
+      ingest_pipeline: 'default',
+    } as RegistryDataStream;
+
+    const { assetsToAdd } = prepareToInstallTemplates(
+      {
+        name: 'package',
+        version: '0.0.1',
+        data_streams: [dataStreamDatasetIsPrefixUnset],
+      } as ArchivePackage,
+      [],
+      []
+    );
+
+    expect(assetsToAdd).not.toContainEqual({ id: 'logs@settings', type: 'component_template' });
+  });
 });
diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
@@ -38,6 +38,7 @@ import {
   FLEET_COMPONENT_TEMPLATES,
   PACKAGE_TEMPLATE_SUFFIX,
   USER_SETTINGS_TEMPLATE_SUFFIX,
+  STACK_COMPONENT_TEMPLATES,
 } from '../../../../constants';
 
 import { getESAssetMetadata } from '../meta';
@@ -530,7 +531,7 @@ export function prepareTemplate({
 
   const isIndexModeTimeSeries =
     dataStream.elasticsearch?.index_mode === 'time_series' ||
-    experimentalDataStreamFeature?.features.tsdb;
+    !!experimentalDataStreamFeature?.features.tsdb;
 
   const validFields = processFields(fields);
 
@@ -572,6 +573,7 @@ export function prepareTemplate({
     registryElasticsearch: dataStream.elasticsearch,
     mappings,
     isIndexModeTimeSeries,
+    type: dataStream.type,
   });
 
   return {
@@ -616,6 +618,8 @@ export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
       .filter(
         (componentTemplateId) => !FLEET_COMPONENT_TEMPLATE_NAMES.includes(componentTemplateId)
       )
+      // Filter stack component templates shared between integrations
+      .filter((componentTemplateId) => !STACK_COMPONENT_TEMPLATES.includes(componentTemplateId))
       .map((componentTemplateId) => ({
         id: componentTemplateId,
         type: ElasticsearchAssetType.componentTemplate,

diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
@@ -57,10 +57,12 @@ describe('EPM template', () => {
 
     const template = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(template.index_patterns).toStrictEqual([templateIndexPattern]);
   });
@@ -69,13 +71,35 @@ describe('EPM template', () => {
     const composedOfTemplates = ['component1', 'component2'];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(template.composed_of).toStrictEqual([
+      'logs@settings',
+      ...composedOfTemplates,
+      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+    ]);
+  });
+
+  it('supplies metrics@tsdb-settings for time series', () => {
+    const composedOfTemplates = ['component1', 'component2'];
+
+    const template = getTemplate({
+      templateIndexPattern: 'metrics-*',
+      type: 'metrics',
+      packageName: 'nginx',
+      composedOfTemplates,
+      templatePriority: 200,
+      mappings: { properties: [] },
+      isIndexModeTimeSeries: true,
+    });
+    expect(template.composed_of).toStrictEqual([
+      'metrics@tsdb-settings',
       ...composedOfTemplates,
       ...FLEET_COMPONENT_TEMPLATES_NAMES,
     ]);
@@ -90,13 +114,16 @@ describe('EPM template', () => {
     const composedOfTemplates = ['component1', 'component2'];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(template.composed_of).toStrictEqual([
+      'logs@settings',
       ...composedOfTemplates,
       FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME,
     ]);
@@ -106,34 +133,43 @@ describe('EPM template', () => {
     const composedOfTemplates: string[] = [];
 
     const template = getTemplate({
-      templateIndexPattern: 'name-*',
+      templateIndexPattern: 'logs-*',
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates,
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
-    expect(template.composed_of).toStrictEqual(FLEET_COMPONENT_TEMPLATES_NAMES);
+    expect(template.composed_of).toStrictEqual([
+      'logs@settings',
+      ...FLEET_COMPONENT_TEMPLATES_NAMES,
+    ]);
   });
 
   it('adds hidden field correctly', () => {
     const templateIndexPattern = 'logs-nginx.access-abcd-*';
 
     const templateWithHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       hidden: true,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithHidden.data_stream.hidden).toEqual(true);
 
     const templateWithoutHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
   });
@@ -143,6 +179,7 @@ describe('EPM template', () => {
 
     const templateWithGlobalAndDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -153,11 +190,13 @@ describe('EPM template', () => {
           hidden: true,
         },
       },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithGlobalAndDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -167,21 +206,25 @@ describe('EPM template', () => {
           hidden: true,
         },
       },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithoutDataStreamHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       hidden: true,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithoutDataStreamHidden.data_stream.hidden).toEqual(true);
 
     const templateWithGlobalHiddenTrueAndDataStreamHiddenFalse = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
@@ -192,15 +235,18 @@ describe('EPM template', () => {
           hidden: false,
         },
       },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithGlobalHiddenTrueAndDataStreamHiddenFalse.data_stream.hidden).toEqual(true);
 
     const templateWithoutHidden = getTemplate({
       templateIndexPattern,
+      type: 'logs',
       packageName: 'nginx',
       composedOfTemplates: [],
       templatePriority: 200,
       mappings: { properties: [] },
+      isIndexModeTimeSeries: false,
     });
     expect(templateWithoutHidden.data_stream.hidden).toEqual(undefined);
   });