From edb3cc33ae91d458a440ede00118f813febf46f1 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 8 Oct 2020 14:39:24 -0400 Subject: [PATCH] query for data_stream constants to create data stream name --- .../ingest_manager/common/types/models/epm.ts | 5 +++ .../epm/elasticsearch/template/template.ts | 45 +++++++++++++++++-- .../epm/kibana/index_pattern/install.ts | 18 ++++---- .../ingest_manager/server/types/index.tsx | 1 + 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index ea7fd60d1fa3f..2ec9d7be6c882 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -44,6 +44,11 @@ export enum ElasticsearchAssetType { transform = 'transform', } +export enum DataType { + logs = 'logs', + metrics = 'metrics', +} + export enum AgentAssetType { input = 'input', } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 561108f96869d..9391474bf4ccb 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -11,6 +11,7 @@ import { TemplateRef, IndexTemplate, IndexTemplateMappings, + DataType, } from '../../../../types'; import { getRegistryDataStreamAssetBaseName } from '../index'; @@ -400,8 +401,6 @@ const updateExistingIndex = async ({ delete mappings.properties.stream; delete mappings.properties.data_stream; - const [, dsType, dsTemplateName, dsNamespace] = indexName.split('-'); - const dataStreamName = `${dsType}-${dsTemplateName}-${dsNamespace}`; // try to update the mappings first try { await callCluster('indices.putMapping', { @@ -411,13 +410,53 @@ const updateExistingIndex = async ({ // if update fails, rollover data stream } catch (err) { try { + // get the data_stream values to compose datastream name + const searchDataStreamFieldsResponse = await callCluster('search', { + index: indexTemplate.index_patterns[0], + body: { + size: 1, + query: { + bool: { + must: [ + { + exists: { + field: 'data_stream.type', + }, + }, + { + exists: { + field: 'data_stream.dataset', + }, + }, + { + exists: { + field: 'data_stream.namespace', + }, + }, + ], + }, + }, + }, + }); + if (searchDataStreamFieldsResponse.hits.total.value === 0) + throw new Error('data_stream fields are missing from datastream indices'); + const { + dataset, + namespace, + type, + }: { + dataset: string; + namespace: string; + type: DataType; + } = searchDataStreamFieldsResponse.hits.hits[0]._source.data_stream; + const dataStreamName = `${type}-${dataset}-${namespace}`; const path = `/${dataStreamName}/_rollover`; await callCluster('transport.request', { method: 'POST', path, }); } catch (error) { - throw new Error(`cannot rollover data stream ${dataStreamName}`); + throw new Error(`cannot rollover data stream ${error}`); } } // update settings after mappings was successful to ensure diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts index 2aa28d23cf857..6238df611642e 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts @@ -12,7 +12,12 @@ import { import * as Registry from '../../registry'; import { loadFieldsFromYaml, Fields, Field } from '../../fields/field'; import { getPackageKeysByStatus } from '../../packages/get'; -import { InstallationStatus, RegistryPackage, CallESAsCurrentUser } from '../../../../types'; +import { + InstallationStatus, + RegistryPackage, + CallESAsCurrentUser, + DataType, +} from '../../../../types'; import { appContextService } from '../../../../services'; interface FieldFormatMap { @@ -69,10 +74,7 @@ export interface IndexPatternField { lang?: string; readFromDocValues: boolean; } -export enum IndexPatternType { - logs = 'logs', - metrics = 'metrics', -} + // TODO: use a function overload and make pkgName and pkgVersion required for install/update // and not for an update removal. or separate out the functions export async function installIndexPatterns( @@ -112,7 +114,7 @@ export async function installIndexPatterns( const installedPackagesInfo = await Promise.all(installedPackagesFetchInfoPromise); // for each index pattern type, create an index pattern - const indexPatternTypes = [IndexPatternType.logs, IndexPatternType.metrics]; + const indexPatternTypes = [DataType.logs, DataType.metrics]; indexPatternTypes.forEach(async (indexPatternType) => { // if this is an update because a package is being unisntalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern if (!pkgName && installedPackages.length === 0) { @@ -140,7 +142,7 @@ export async function installIndexPatterns( // of all fields from all data streams matching data stream type export const getAllDataStreamFieldsByType = async ( packages: RegistryPackage[], - dataStreamType: IndexPatternType + dataStreamType: DataType ): Promise => { const dataStreamsPromises = packages.reduce>>((acc, pkg) => { if (pkg.data_streams) { @@ -385,7 +387,7 @@ export const ensureDefaultIndices = async (callCluster: CallESAsCurrentUser) => // that no matching indices exist https://github.com/elastic/kibana/issues/62343 const logger = appContextService.getLogger(); return Promise.all( - Object.keys(IndexPatternType).map(async (indexPattern) => { + Object.keys(DataType).map(async (indexPattern) => { const defaultIndexPatternName = indexPattern + INDEX_PATTERN_PLACEHOLDER_SUFFIX; const indexExists = await callCluster('indices.exists', { index: defaultIndexPatternName }); if (!indexExists) { diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 0c070959e3b93..7d841ed024ce5 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -73,6 +73,7 @@ export { // Agent Request types PostAgentEnrollRequest, PostAgentCheckinRequest, + DataType, } from '../../common'; export type CallESAsCurrentUser = LegacyScopedClusterClient['callAsCurrentUser'];