diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts index 20a83be0573b4..665055132df37 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts @@ -69,9 +69,9 @@ describe('Ingest Manager - packageToPackagePolicy', () => { packageToPackagePolicyInputs(({ ...mockPackage, data_streams: [ - { type: 'logs', name: 'foo', streams: [{ input: 'foo' }] }, - { type: 'logs', name: 'bar', streams: [{ input: 'bar' }] }, - { type: 'logs', name: 'bar2', streams: [{ input: 'bar' }] }, + { type: 'logs', dataset: 'foo', streams: [{ input: 'foo' }] }, + { type: 'logs', dataset: 'bar', streams: [{ input: 'bar' }] }, + { type: 'logs', dataset: 'bar2', streams: [{ input: 'bar' }] }, ], config_templates: [ { @@ -105,12 +105,12 @@ describe('Ingest Manager - packageToPackagePolicy', () => { data_streams: [ { type: 'logs', - name: 'foo', + dataset: 'foo', streams: [{ input: 'foo', vars: [{ default: 'foo-var-value', name: 'var-name' }] }], }, { type: 'logs', - name: 'bar', + dataset: 'bar', streams: [ { input: 'bar', @@ -120,7 +120,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, { type: 'logs', - name: 'bar2', + dataset: 'bar2', streams: [ { input: 'bar', @@ -176,12 +176,12 @@ describe('Ingest Manager - packageToPackagePolicy', () => { data_streams: [ { type: 'logs', - name: 'foo', + dataset: 'foo', streams: [{ input: 'foo', vars: [{ default: 'foo-var-value', name: 'var-name' }] }], }, { type: 'logs', - name: 'bar', + dataset: 'bar', streams: [ { input: 'bar', @@ -191,7 +191,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, { type: 'logs', - name: 'bar2', + dataset: 'bar2', streams: [ { input: 'bar', @@ -201,7 +201,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, { type: 'logs', - name: 'disabled', + dataset: 'disabled', streams: [ { input: 'with-disabled-streams', @@ -212,7 +212,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => { }, { type: 'logs', - name: 'disabled2', + dataset: 'disabled2', streams: [ { input: 'with-disabled-streams', diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts index 3a83cde0686e5..0418b97922f52 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_package_policy.ts @@ -29,7 +29,7 @@ const getStreamsForInputType = ( ...stream, data_stream: { type: dataStream.type, - dataset: dataStream.name, + dataset: dataStream.dataset, }, }); } diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 5f3d631fb844a..1e123f315be4e 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -176,7 +176,7 @@ export type ElasticsearchAssetTypeToParts = Record< export interface RegistryDataStream { type: string; - name: string; + dataset: string; title: string; release: string; streams?: RegistryStream[]; diff --git a/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md b/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md index fd7edcb7fcca0..42a0bbc218869 100644 --- a/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md +++ b/x-pack/plugins/ingest_manager/dev_docs/indexing_strategy.md @@ -6,48 +6,48 @@ Overall documentation of Ingest Management is now maintained in the `elastic/sta Ingest Management enforces an indexing strategy to allow the system to automatically detect indices and run queries on it. In short the indexing strategy looks as following: ``` -{dataset.type}-{dataset.name}-{dataset.namespace} +{data_stream.type}-{data_stream.dataset}-{data_stream.namespace} ``` -The `{dataset.type}` can be `logs` or `metrics`. The `{dataset.namespace}` is the part where the user can use free form. The only two requirement are that it has only characters allowed in an Elasticsearch index name and does NOT contain a `-`. The `dataset` is defined by the data that is indexed. The same requirements as for the namespace apply. It is expected that the fields for type, namespace and dataset are part of each event and are constant keywords. If there is a dataset or a namespace with a `-` inside, it is recommended to replace it either by a `.` or a `_`. +The `{data_stream.type}` can be `logs` or `metrics`. The `{data_stream.namespace}` is the part where the user can use free form. The only two requirement are that it has only characters allowed in an Elasticsearch index name and does NOT contain a `-`. The `data_stream` is defined by the data that is indexed. The same requirements as for the namespace apply. It is expected that the fields for type, dataset, and namespace are part of each event and are constant keywords. If there is a dataset or a namespace with a `-` inside, it is recommended to replace it either by a `.` or a `_`. -Note: More `{dataset.type}`s might be added in the future like `traces`. +Note: More `{data_stream.type}`s might be added in the future like `traces`. This indexing strategy has a few advantages: -* Each index contains only the fields which are relevant for the dataset. This leads to more dense indices and better field completion. -* ILM policies can be applied per namespace per dataset. -* Rollups can be specified per namespace per dataset. -* Having the namespace user configurable makes setting security permissions possible. -* Having a global metrics and logs template, allows to create new indices on demand which still follow the convention. This is common in the case of k8s as an example. -* Constant keywords allow to narrow down the indices we need to access for querying very efficiently. This is especially relevant in environments which a large number of indices or with indices on slower nodes. +- Each index contains only the fields which are relevant for the datta stream. This leads to more dense indices and better field completion. +- ILM policies can be applied per namespace per data stream. +- Rollups can be specified per namespace per data stream. +- Having the namespace user configurable makes setting security permissions possible. +- Having a global metrics and logs template, allows to create new indices on demand which still follow the convention. This is common in the case of k8s as an example. +- Constant keywords allow to narrow down the indices we need to access for querying very efficiently. This is especially relevant in environments which a large number of indices or with indices on slower nodes. Overall it creates smaller indices in size, makes querying more efficient and allows users to define their own naming parts in namespace and still benefiting from all features that can be built on top of the indexing startegy. ## Ingest Pipeline -The ingest pipelines for a specific dataset will have the following naming scheme: +The ingest pipelines for a specific data stream will have the following naming scheme: ``` -{dataset.type}-{dataset.name}-{package.version} +{data_stream.type}-{data_stream.dataset}-{package.version} ``` -As an example, the ingest pipeline for the Nginx access logs is called `logs-nginx.access-3.4.1`. The same ingest pipeline is used for all namespaces. It is possible that a dataset has multiple ingest pipelines in which case a suffix is added to the name. +As an example, the ingest pipeline for the Nginx access logs is called `logs-nginx.access-3.4.1`. The same ingest pipeline is used for all namespaces. It is possible that a data stream has multiple ingest pipelines in which case a suffix is added to the name. The version is included in each pipeline to allow upgrades. The pipeline itself is listed in the index template and is automatically applied at ingest time. ## Templates & ILM Policies -To make the above strategy possible, alias templates are required. For each type there is a basic alias template with a default ILM policy. These default templates apply to all indices which follow the indexing strategy and do not have a more specific dataset alias template. +To make the above strategy possible, alias templates are required. For each type there is a basic alias template with a default ILM policy. These default templates apply to all indices which follow the indexing strategy and do not have a more specific data stream alias template. The `metrics` and `logs` alias template contain all the basic fields from ECS. Each type template contains an ILM policy. Modifying this default ILM policy will affect all data covered by the default templates. -The templates for a dataset are called as following: +The templates for a data stream are called as following: ``` -{dataset.type}-{dataset.name} +{data_stream.type}-{data_stream.dataset} ``` The pattern used inside the index template is `{type}-{dataset}-*` to match all namespaces. diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts index 10f244a92d17b..807684518f6db 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.test..ts @@ -34,7 +34,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { status: InstallationStatus.notInstalled, data_streams: [ { - name: 'foo', + dataset: 'foo', streams: [ { input: 'foo', @@ -44,7 +44,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { ], }, { - name: 'bar', + dataset: 'bar', streams: [ { input: 'bar', @@ -59,7 +59,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { ], }, { - name: 'bar2', + dataset: 'bar2', streams: [ { input: 'bar', @@ -69,7 +69,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { ], }, { - name: 'disabled', + dataset: 'disabled', streams: [ { input: 'with-disabled-streams', @@ -80,7 +80,7 @@ describe('Ingest Manager - validatePackagePolicy()', () => { ], }, { - name: 'disabled2', + dataset: 'disabled2', streams: [ { input: 'with-disabled-streams', diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts index 95451e80fa9f3..fdc0bfeeb201f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/services/validate_package_policy.ts @@ -92,10 +92,10 @@ export const validatePackagePolicy = ( return inputs; }, {} as Record); - const registryStreamsByDataStream: Record = ( + const registryStreamsByDataset: Record = ( packageInfo.data_streams || [] ).reduce((dataStreams, registryDataStream) => { - dataStreams[registryDataStream.name] = registryDataStream.streams || []; + dataStreams[registryDataStream.dataset] = registryDataStream.streams || []; return dataStreams; }, {} as Record); @@ -140,7 +140,7 @@ export const validatePackagePolicy = ( if (stream.vars) { const streamVarsByName = ( ( - registryStreamsByDataStream[stream.data_stream.dataset].find( + registryStreamsByDataset[stream.data_stream.dataset].find( (registryStream) => registryStream.input === input.type ) || {} ).vars || [] diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx index 78c2bebab0044..aa63b0fdd16f8 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/create_package_policy_page/step_configure_package.tsx @@ -23,7 +23,7 @@ const findStreamsForInputType = ( streams.push({ ...stream, data_stream: { - dataset: dataStream.name, + dataset: dataStream.dataset, }, }); } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.test.ts index 9ea97873949f6..78aa17da5030c 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.test.ts @@ -9,7 +9,7 @@ import { getRegistryDataStreamAssetBaseName } from './index'; test('getBaseName', () => { const dataStream: RegistryDataStream = { - name: 'nginx.access', + dataset: 'nginx.access', title: 'Nginx Acess Logs', release: 'beta', type: 'logs', diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.ts index 326244d4daa1c..17cd28cc8a081 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/index.ts @@ -8,8 +8,8 @@ import { RegistryDataStream } from '../../../types'; /** * Creates the base name for Elasticsearch assets in the form of - * {type}-{id} + * {type}-{dataset} */ export function getRegistryDataStreamAssetBaseName(dataStream: RegistryDataStream): string { - return `${dataStream.type}-${dataStream.name}`; + return `${dataStream.type}-${dataStream.dataset}`; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts index 876f6bea50fc9..378dd271779b4 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/ingest_pipelines.test.ts @@ -107,7 +107,7 @@ test('a yml-format pipeline with no pipeline references stays unchanged', () => test('getPipelineNameForInstallation gets correct name', () => { const dataStream: RegistryDataStream = { - name: 'coredns.log', + dataset: 'coredns.log', title: 'CoreDNS logs', release: 'ga', type: 'logs', @@ -128,9 +128,9 @@ test('getPipelineNameForInstallation gets correct name', () => { packageVersion, }); expect(pipelineEntryNameForInstallation).toBe( - `${dataStream.type}-${dataStream.name}-${packageVersion}` + `${dataStream.type}-${dataStream.dataset}-${packageVersion}` ); expect(pipelineRefNameForInstallation).toBe( - `${dataStream.type}-${dataStream.name}-${packageVersion}-${pipelineRefName}` + `${dataStream.type}-${dataStream.dataset}-${packageVersion}-${pipelineRefName}` ); }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts index b87eb1d90f8e9..ac85344679ccb 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -218,5 +218,5 @@ export const getPipelineNameForInstallation = ({ const isPipelineEntry = pipelineName === dataStream.ingest_pipeline; const suffix = isPipelineEntry ? '' : `-${pipelineName}`; // if this is the pipeline entry, don't add a suffix - return `${dataStream.type}-${dataStream.name}-${packageVersion}${suffix}`; + return `${dataStream.type}-${dataStream.dataset}-${packageVersion}${suffix}`; }; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts index 6d8214c8b05d9..37c27a8ad477d 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts @@ -167,5 +167,5 @@ const getTransformNameForInstallation = ( suffix: string ) => { const filename = transformDataStream?.path.split('/')?.pop()?.split('.')[0]; - return `${transformDataStream.data_stream.type}-${transformDataStream.data_stream.name}-${filename}-${suffix}`; + return `${transformDataStream.data_stream.type}-${transformDataStream.data_stream.dataset}-${filename}-${suffix}`; }; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts index 06f5143c7984a..02b505e1b123e 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts @@ -98,7 +98,7 @@ describe('test transform install', () => { data_streams: [ { type: 'metrics', - name: 'endpoint.metadata', + dataset: 'endpoint.metadata', title: 'Endpoint Metadata', release: 'experimental', package: 'endpoint', @@ -112,7 +112,7 @@ describe('test transform install', () => { }, { type: 'metrics', - name: 'endpoint.metadata_current', + dataset: 'endpoint.metadata_current', title: 'Endpoint Metadata Current', release: 'experimental', package: 'endpoint', @@ -270,7 +270,7 @@ describe('test transform install', () => { data_streams: [ { type: 'metrics', - name: 'endpoint.metadata_current', + dataset: 'endpoint.metadata_current', title: 'Endpoint Metadata', release: 'experimental', package: 'endpoint', @@ -353,7 +353,7 @@ describe('test transform install', () => { data_streams: [ { type: 'metrics', - name: 'endpoint.metadata', + dataset: 'endpoint.metadata', title: 'Endpoint Metadata', release: 'experimental', package: 'endpoint', @@ -367,7 +367,7 @@ describe('test transform install', () => { }, { type: 'metrics', - name: 'endpoint.metadata_current', + dataset: 'endpoint.metadata_current', title: 'Endpoint Metadata Current', release: 'experimental', package: 'endpoint', diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts index 2f60c74d3514f..df970e629f502 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.test.ts @@ -17,7 +17,7 @@ const mockInstallation: SavedObject = { installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], es_index_patterns: { pattern: 'pattern-name' }, - name: 'test packagek', + name: 'test package', version: '1.0.0', install_status: 'installed', install_version: '1.0.0', @@ -33,7 +33,7 @@ const mockInstallationUpdateFail: SavedObject = { installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }], installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }], es_index_patterns: { pattern: 'pattern-name' }, - name: 'test packagek', + name: 'test package', version: '1.0.0', install_status: 'installing', install_version: '1.0.1', diff --git a/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts b/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts index 93c1ab9fdec2a..7100935ff0b64 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_policy.test.ts @@ -11,12 +11,12 @@ import { PackageInfo, PackagePolicySOAttributes } from '../types'; import { SavedObjectsUpdateResponse } from 'src/core/server'; async function mockedGetAssetsData(_a: any, _b: any, dataset: string) { - if (dataset === 'datastream1') { + if (dataset === 'dataset1') { return [ { buffer: Buffer.from(` type: log -metricset: ["datastream1"] +metricset: ["dataset1"] paths: {{#each paths}} - {{this}} @@ -48,7 +48,7 @@ describe('Package policy service', () => { data_streams: [ { type: 'logs', - name: 'package.datastream1', + dataset: 'package.dataset1', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], }, ], @@ -65,7 +65,7 @@ describe('Package policy service', () => { streams: [ { id: 'datastream01', - data_stream: { dataset: 'package.datastream1', type: 'logs' }, + data_stream: { dataset: 'package.dataset1', type: 'logs' }, enabled: true, vars: { paths: { @@ -85,7 +85,7 @@ describe('Package policy service', () => { streams: [ { id: 'datastream01', - data_stream: { dataset: 'package.datastream1', type: 'logs' }, + data_stream: { dataset: 'package.dataset1', type: 'logs' }, enabled: true, vars: { paths: { @@ -93,7 +93,7 @@ describe('Package policy service', () => { }, }, compiled_stream: { - metricset: ['datastream1'], + metricset: ['dataset1'], paths: ['/var/log/set.log'], type: 'log', }, @@ -108,7 +108,7 @@ describe('Package policy service', () => { ({ data_streams: [ { - name: 'package.datastream1', + dataset: 'package.dataset1', type: 'logs', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], }, @@ -131,7 +131,7 @@ describe('Package policy service', () => { streams: [ { id: 'datastream01', - data_stream: { dataset: 'package.datastream1', type: 'logs' }, + data_stream: { dataset: 'package.dataset1', type: 'logs' }, enabled: true, }, ], @@ -151,10 +151,10 @@ describe('Package policy service', () => { streams: [ { id: 'datastream01', - data_stream: { dataset: 'package.datastream1', type: 'logs' }, + data_stream: { dataset: 'package.dataset1', type: 'logs' }, enabled: true, compiled_stream: { - metricset: ['datastream1'], + metricset: ['dataset1'], paths: ['/var/log/set.log'], type: 'log', }, diff --git a/x-pack/plugins/ingest_manager/server/services/package_policy.ts b/x-pack/plugins/ingest_manager/server/services/package_policy.ts index 0eaead6608fa8..d91f6e8580fc3 100644 --- a/x-pack/plugins/ingest_manager/server/services/package_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/package_policy.ts @@ -381,7 +381,7 @@ async function _assignPackageStreamToStream( } const packageDataStream = packageDataStreams.find( - (pkgDataStream) => pkgDataStream.name === stream.data_stream.dataset + (pkgDataStream) => pkgDataStream.dataset === stream.data_stream.dataset ); if (!packageDataStream) { throw new Error(`Stream template not found, unable to find dataset ${datasetPath}`);