[EPM][Security Solution] Implementing dataset component templates (#7…

…0517) (#70862)

* Implementing dataset component templates

* Fixing test

* Temporary fix to include timestamp with any component template created

* Update package registry docker image for CI.

* Adapt to new registry filesystem layout.

* Adjust tests to changed registry behavior.

* Adding a test for mappings and settings overrides

* Wrap all the tests in the docker check

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Sonja Krause-Harder <[email protected]>

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Sonja Krause-Harder <[email protected]>

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -175,6 +175,12 @@ export interface Dataset {
   package: string;
   path: string;
   ingest_pipeline: string;
+  elasticsearch?: RegistryElasticsearch;
+}
+
+export interface RegistryElasticsearch {
+  'index_template.settings'?: object;
+  'index_template.mappings'?: object;
 }
 
 // EPR types this as `[]map[string]interface{}`
@@ -272,6 +278,7 @@ export interface IndexTemplate {
   data_stream: {
     timestamp_field: string;
   };
+  composed_of: string[];
   _meta: object;
 }
 

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts

@@ -5,7 +5,13 @@
  */
 
 import Boom from 'boom';
-import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types';
+import {
+  Dataset,
+  RegistryPackage,
+  ElasticsearchAssetType,
+  TemplateRef,
+  RegistryElasticsearch,
+} from '../../../../types';
 import { CallESAsCurrentUser } from '../../../../types';
 import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
@@ -157,6 +163,98 @@ export async function installTemplateForDataset({
   });
 }
 
+function putComponentTemplate(
+  body: object | undefined,
+  name: string,
+  callCluster: CallESAsCurrentUser
+): { clusterPromise: Promise<any>; name: string } | undefined {
+  if (body) {
+    const callClusterParams: {
+      method: string;
+      path: string;
+      ignore: number[];
+      body: any;
+    } = {
+      method: 'PUT',
+      path: `/_component_template/${name}`,
+      ignore: [404],
+      body,
+    };
+
+    return { clusterPromise: callCluster('transport.request', callClusterParams), name };
+  }
+}
+
+function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
+  let mappingsTemplate;
+  let settingsTemplate;
+
+  if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
+    mappingsTemplate = {
+      template: {
+        mappings: {
+          ...registryElasticsearch['index_template.mappings'],
+          // temporary change until https://github.com/elastic/elasticsearch/issues/58956 is resolved
+          // hopefully we'll be able to remove the entire properties section once that issue is resolved
+          properties: {
+            // if the timestamp_field changes here: https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts#L309
+            // we'll need to update this as well
+            '@timestamp': {
+              type: 'date',
+            },
+          },
+        },
+      },
+    };
+  }
+
+  if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
+    settingsTemplate = {
+      template: {
+        settings: registryElasticsearch['index_template.settings'],
+      },
+    };
+  }
+  return { settingsTemplate, mappingsTemplate };
+}
+
+async function installDatasetComponentTemplates(
+  templateName: string,
+  registryElasticsearch: RegistryElasticsearch | undefined,
+  callCluster: CallESAsCurrentUser
+) {
+  const templates: string[] = [];
+  const componentPromises: Array<Promise<any>> = [];
+
+  const compTemplates = buildComponentTemplates(registryElasticsearch);
+
+  const mappings = putComponentTemplate(
+    compTemplates.mappingsTemplate,
+    `${templateName}-mappings`,
+    callCluster
+  );
+
+  const settings = putComponentTemplate(
+    compTemplates.settingsTemplate,
+    `${templateName}-settings`,
+    callCluster
+  );
+
+  if (mappings) {
+    templates.push(mappings.name);
+    componentPromises.push(mappings.clusterPromise);
+  }
+
+  if (settings) {
+    templates.push(settings.name);
+    componentPromises.push(settings.clusterPromise);
+  }
+
+  // TODO: Check return values for errors
+  await Promise.all(componentPromises);
+  return templates;
+}
+
 export async function installTemplate({
   callCluster,
   fields,
@@ -180,13 +278,22 @@ export async function installTemplate({
       packageVersion,
     });
   }
+
+  const composedOfTemplates = await installDatasetComponentTemplates(
+    templateName,
+    dataset.elasticsearch,
+    callCluster
+  );
+
   const template = getTemplate({
     type: dataset.type,
     templateName,
     mappings,
     pipelineName,
     packageName,
+    composedOfTemplates,
   });
+
   // TODO: Check return values for errors
   const callClusterParams: {
     method: string;

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts

@@ -29,10 +29,37 @@ test('get template', () => {
     templateName,
     packageName: 'nginx',
     mappings: { properties: {} },
+    composedOfTemplates: [],
   });
   expect(template.index_patterns).toStrictEqual([`${templateName}-*`]);
 });
 
+test('adds composed_of correctly', () => {
+  const composedOfTemplates = ['component1', 'component2'];
+
+  const template = getTemplate({
+    type: 'logs',
+    templateName: 'name',
+    packageName: 'nginx',
+    mappings: { properties: {} },
+    composedOfTemplates,
+  });
+  expect(template.composed_of).toStrictEqual(composedOfTemplates);
+});
+
+test('adds empty composed_of correctly', () => {
+  const composedOfTemplates: string[] = [];
+
+  const template = getTemplate({
+    type: 'logs',
+    templateName: 'name',
+    packageName: 'nginx',
+    mappings: { properties: {} },
+    composedOfTemplates,
+  });
+  expect(template.composed_of).toStrictEqual(composedOfTemplates);
+});
+
 test('tests loading base.yml', () => {
   const ymlPath = path.join(__dirname, '../../fields/tests/base.yml');
   const fieldsYML = readFileSync(ymlPath, 'utf-8');
@@ -45,6 +72,7 @@ test('tests loading base.yml', () => {
     templateName: 'foo',
     packageName: 'nginx',
     mappings,
+    composedOfTemplates: [],
   });
 
   expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -62,6 +90,7 @@ test('tests loading coredns.logs.yml', () => {
     templateName: 'foo',
     packageName: 'coredns',
     mappings,
+    composedOfTemplates: [],
   });
 
   expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -79,6 +108,7 @@ test('tests loading system.yml', () => {
     templateName: 'whatsthis',
     packageName: 'system',
     mappings,
+    composedOfTemplates: [],
   });
 
   expect(template).toMatchSnapshot(path.basename(ymlPath));

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -43,14 +43,16 @@ export function getTemplate({
   mappings,
   pipelineName,
   packageName,
+  composedOfTemplates,
 }: {
   type: string;
   templateName: string;
   mappings: IndexTemplateMappings;
   pipelineName?: string | undefined;
   packageName: string;
+  composedOfTemplates: string[];
 }): IndexTemplate {
-  const template = getBaseTemplate(type, templateName, mappings, packageName);
+  const template = getBaseTemplate(type, templateName, mappings, packageName, composedOfTemplates);
   if (pipelineName) {
     template.template.settings.index.default_pipeline = pipelineName;
   }
@@ -244,7 +246,8 @@ function getBaseTemplate(
   type: string,
   templateName: string,
   mappings: IndexTemplateMappings,
-  packageName: string
+  packageName: string,
+  composedOfTemplates: string[]
 ): IndexTemplate {
   return {
     // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*)
@@ -308,6 +311,7 @@ function getBaseTemplate(
     data_stream: {
       timestamp_field: '@timestamp',
     },
+    composed_of: composedOfTemplates,
     _meta: {
       package: {
         name: packageName,

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -41,6 +41,7 @@ export {
   PackageInfo,
   RegistryVarsEntry,
   Dataset,
+  RegistryElasticsearch,
   AssetReference,
   ElasticsearchAssetType,
   IngestAssetType,

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: dataset.type
+  type: constant_keyword
+  description: >
+    Dataset type.
+- name: dataset.name
+  type: constant_keyword
+  description: >
+    Dataset name.
+- name: dataset.namespace
+  type: constant_keyword
+  description: >
+    Dataset namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml

@@ -0,0 +1,9 @@
+title: Test Dataset
+
+type: logs
+
+elasticsearch:
+  index_template.mappings:
+    dynamic: false
+  index_template.settings:
+    index.lifecycle.name: reference

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+For testing the that the settings and mappings section get used

x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml

@@ -0,0 +1,20 @@
+format_version: 1.0.0
+name: overrides
+title: Mappings Settings Test
+description: This is a test package for testing that the mappings and settings sections in the dataset manifest are applied.
+version: 0.1.0
+categories: ['security']
+release: beta
+type: integration
+license: basic
+
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+
+icons:
+  - src: '/img/logo_overrides_64_color.svg'
+    size: '16x16'
+    type: 'image/svg+xml'

x-pack/test/ingest_manager_api_integration/apis/index.js

@@ -11,5 +11,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./file'));
     //loadTestFile(require.resolve('./template'));
     loadTestFile(require.resolve('./ilm'));
+    loadTestFile(require.resolve('./install'));
   });
 }