From 3be93b3d5a36e950af0d7f1a753ca3ab05d7acc4 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Mon, 28 Jan 2019 09:51:13 -0800 Subject: [PATCH 1/7] Support for reindexing APM indices Signed-off-by: Tyler Smalley --- src/legacy/core_plugins/apm_oss/index.d.ts | 22 + src/legacy/core_plugins/apm_oss/index.js | 18 +- .../core_plugins/elasticsearch/index.d.ts | 8 +- src/server/kbn_server.d.ts | 3 + .../plugins/upgrade_assistant/common/types.ts | 1 + x-pack/plugins/upgrade_assistant/index.ts | 4 +- .../tabs/checkup/deprecations/index_table.tsx | 2 +- .../tabs/checkup/deprecations/list.tsx | 7 +- .../reindex/flyout/warnings_step.tsx | 29 + .../server/lib/apm/index.test.ts | 117 ++ .../upgrade_assistant/server/lib/apm/index.ts | 353 ++++ .../server/lib/apm/mapping.json | 1600 +++++++++++++++++ .../server/lib/es_migration_apis.test.ts | 4 +- .../server/lib/es_migration_apis.ts | 73 +- .../server/lib/reindexing/index_settings.ts | 16 +- .../server/lib/reindexing/reindex_service.ts | 30 +- .../server/lib/reindexing/types.ts | 5 + .../server/lib/reindexing/worker.ts | 10 +- .../server/routes/cluster_checkup.ts | 9 +- .../server/routes/reindex_indices.ts | 15 +- 20 files changed, 2272 insertions(+), 54 deletions(-) create mode 100644 src/legacy/core_plugins/apm_oss/index.d.ts create mode 100644 x-pack/plugins/upgrade_assistant/server/lib/apm/index.test.ts create mode 100644 x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts create mode 100644 x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json diff --git a/src/legacy/core_plugins/apm_oss/index.d.ts b/src/legacy/core_plugins/apm_oss/index.d.ts new file mode 100644 index 0000000000000..86fe4e0350dce --- /dev/null +++ b/src/legacy/core_plugins/apm_oss/index.d.ts @@ -0,0 +1,22 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export interface ApmOssPlugin { + indexPatterns: string[]; +} diff --git a/src/legacy/core_plugins/apm_oss/index.js b/src/legacy/core_plugins/apm_oss/index.js index 72683bed8970e..2c404b6197f5a 100644 --- a/src/legacy/core_plugins/apm_oss/index.js +++ b/src/legacy/core_plugins/apm_oss/index.js @@ -17,6 +17,8 @@ * under the License. */ +import _ from 'lodash'; + export default function apmOss(kibana) { return new kibana.Plugin({ id: 'apm_oss', @@ -30,12 +32,24 @@ export default function apmOss(kibana) { indexPattern: Joi.string().default('apm-*'), // ES Indices + sourcemapIndices: Joi.string().default('apm-*'), errorIndices: Joi.string().default('apm-*'), - onboardingIndices: Joi.string().default('apm-*'), - spanIndices: Joi.string().default('apm-*'), transactionIndices: Joi.string().default('apm-*'), + spanIndices: Joi.string().default('apm-*'), metricsIndices: Joi.string().default('apm-*'), + onboardingIndices: Joi.string().default('apm-*'), }).default(); }, + + init(server) { + server.expose('indexPatterns', _.uniq([ + 'sourcemapIndices', + 'errorIndices', + 'transactionIndices', + 'spanIndices', + 'metricsIndices', + 'onboardingIndices' + ].map(type => server.config().get(`apm_oss.${type}`)))); + } }); } diff --git a/src/legacy/core_plugins/elasticsearch/index.d.ts b/src/legacy/core_plugins/elasticsearch/index.d.ts index 207ea48dcdf1c..5835fe6e3d5e3 100644 --- a/src/legacy/core_plugins/elasticsearch/index.d.ts +++ b/src/legacy/core_plugins/elasticsearch/index.d.ts @@ -202,12 +202,14 @@ export interface DeprecationInfo { details?: string; } +export interface IndexSettingsDeprecationInfo { + [indexName: string]: DeprecationInfo[]; +} + export interface DeprecationAPIResponse { cluster_settings: DeprecationInfo[]; node_settings: DeprecationInfo[]; - index_settings: { - [indexName: string]: DeprecationInfo[]; - }; + index_settings: IndexSettingsDeprecationInfo; } export interface CallClusterOptions { diff --git a/src/server/kbn_server.d.ts b/src/server/kbn_server.d.ts index 9ec8a6349045d..5bf7657dcafd8 100644 --- a/src/server/kbn_server.d.ts +++ b/src/server/kbn_server.d.ts @@ -19,7 +19,9 @@ import { Server } from 'hapi'; +import { ApmOssPlugin } from '../legacy/core_plugins/apm_oss'; import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch'; + import { IndexPatternsServiceFactory } from './index_patterns'; import { SavedObjectsClient, SavedObjectsService } from './saved_objects'; @@ -33,6 +35,7 @@ declare module 'hapi' { elasticsearch: ElasticsearchPlugin; kibana: any; spaces: any; + apm_oss: ApmOssPlugin; // add new plugin types here } diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts index 4dc6917ecd7ab..ed8085eadf740 100644 --- a/x-pack/plugins/upgrade_assistant/common/types.ts +++ b/x-pack/plugins/upgrade_assistant/common/types.ts @@ -49,4 +49,5 @@ export enum ReindexWarning { booleanFields = 1, // 7.0 -> 8.0 warnings + apmReindex, } diff --git a/x-pack/plugins/upgrade_assistant/index.ts b/x-pack/plugins/upgrade_assistant/index.ts index e8f03767dc49d..956dedf35d463 100644 --- a/x-pack/plugins/upgrade_assistant/index.ts +++ b/x-pack/plugins/upgrade_assistant/index.ts @@ -3,8 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { Server } from 'hapi'; import Joi from 'joi'; +import { Legacy } from 'kibana'; import { resolve } from 'path'; import { initServer } from './server'; @@ -30,7 +30,7 @@ export function upgradeAssistant(kibana: any) { }).default(); }, - init(server: Server) { + init(server: Legacy.Server) { // Add server routes and initialize the plugin here initServer(server); }, diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx index 5deed2523e215..d06647038e05f 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx @@ -135,7 +135,7 @@ export class IndexDeprecationTableUI extends React.Component< // NOTE: this naive implementation assumes all indices in the table are // should show the reindex button. This should work for known usecases. const { indices } = this.props; - if (!indices.find(i => i.reindex)) { + if (!indices.find(i => i.reindex === true)) { return null; } diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx index 0783802b6c0c6..db838ce137c6f 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx @@ -10,13 +10,10 @@ import { DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch'; import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis'; import { GroupByOption } from '../../../types'; -import { CURRENT_MAJOR_VERSION } from 'x-pack/plugins/upgrade_assistant/common/version'; import { COLOR_MAP, LEVEL_MAP } from '../constants'; import { DeprecationCell } from './cell'; import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table'; -const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`; - const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => { return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]); }; @@ -37,7 +34,7 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI @@ -91,7 +88,7 @@ export const DeprecationList: StatelessComponent<{ const indices = deprecations.map(dep => ({ index: dep.index!, details: dep.details, - reindex: dep.message === OLD_INDEX_MESSAGE, + reindex: dep.reindex, })); return ; diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx index 8a4c2eb7bf516..ff45ce1596add 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx @@ -107,6 +107,35 @@ export class WarningsFlyoutStep extends React.Component< + {warnings.includes(ReindexWarning.apmReindex) && ( + + This index will be converted to ECS format} + checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]} + onChange={this.onChange} + /> +

+ Starting in version 7.0.0, APM data will be represented in the Elastic Common + Schema. In order for legacy data to be included, it is required that the data is + re-indexed to support this new format. +
+ + Documentation + +
+ + More about ECS + +

+ + )} + + + {warnings.includes(ReindexWarning.booleanFields) && ( { + return { + 'foo-1': { + mappings: {}, + }, + 'foo-2': { + mappings: { + _meta: { + version: '6.7.0', + }, + }, + }, + 'foo-3': { + mappings: { + _meta: { + version: '7.0.0', + }, + }, + }, + 'foo-4': { + mappings: { + _meta: { + version: '7.1.0', + }, + }, + }, + }; + }); +} + +describe('getDeprecatedApmIndices', () => { + it('calls indices.getMapping', async () => { + const callWithRequest = mockedCallWithRequest(); + await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*', 'bar-*']); + + expect(callWithRequest).toHaveBeenCalledWith({}, 'indices.getMapping', { + index: 'foo-*,bar-*', + filterPath: '*.mappings._meta.version,*.mappings.properties.@timestamp', + }); + }); + + it('includes mappings not yet at 7.0.0', async () => { + const callWithRequest = mockedCallWithRequest(); + const deprecations = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']); + + expect(deprecations).toHaveLength(2); + expect(deprecations[0].index).toEqual('foo-1'); + expect(deprecations[1].index).toEqual('foo-2'); + }); + + it('formats the deprecations', async () => { + const callWithRequest = mockedCallWithRequest(); + // @ts-ignore + const [deprecation, _] = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']); + + expect(deprecation.level).toEqual('critical'); + expect(deprecation.message).toEqual('APM index needs converted to ECS format'); + expect(deprecation.url).toEqual( + 'https://www.elastic.co/guide/en/apm/server/master/breaking-changes.html' + ); + expect(deprecation.details).toEqual('This index was created prior to 7.0.'); + expect(deprecation.reindex).toBe(true); + }); +}); + +describe.only('isLegacyApmIndex', () => { + it('is true when for no version', () => { + expect(isLegacyApmIndex('foo-1', ['foo-*'], {})).toEqual(true); + }); + + it('is true when version is less than 7.0.0', () => { + expect( + isLegacyApmIndex('foo-1', ['foo-*'], { + _meta: { version: '6.7.0' }, + }) + ).toEqual(true); + }); + + it('is false when version is 7.0.0', () => { + expect( + isLegacyApmIndex('foo-1', ['foo-*'], { + _meta: { version: '7.0.0' }, + }) + ).toEqual(false); + }); + + it('is false when version is greater than 7.0.0', () => { + expect( + isLegacyApmIndex('foo-1', ['foo-*'], { + _meta: { version: '7.1.0' }, + }) + ).toEqual(false); + }); + + it('handles multiple index patterns', () => { + expect( + isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], { + _meta: { version: '6.7.0' }, + }) + ).toEqual(true); + + expect( + isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], { + _meta: { version: '7.0.0' }, + }) + ).toEqual(false); + }); +}); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts new file mode 100644 index 0000000000000..33eb988dd34fc --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request } from 'hapi'; +import { get } from 'lodash'; +import minimatch from 'minimatch'; +import semver from 'semver'; +import { CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch'; +import pkg from '../../../../../package.json'; + +import { EnrichedDeprecationInfo } from '../es_migration_apis'; +import { FlatSettings } from '../reindexing/types'; + +export async function getDeprecatedApmIndices( + callWithRequest: CallClusterWithRequest, + request: Request, + indexPatterns: string[] = [] +): Promise { + const indices = await callWithRequest(request, 'indices.getMapping', { + index: indexPatterns.join(','), + // we include @timestamp to prevent filtering mappings without a version + // since @timestamp is expected to always exist + filterPath: '*.mappings._meta.version,*.mappings.properties.@timestamp', + }); + + return Object.keys(indices).reduce((deprecations: EnrichedDeprecationInfo[], index) => { + if (semver.lt(get(indices[index], 'mappings._meta.version', '0.0.0'), pkg.version)) { + deprecations.push({ + level: 'critical', + message: 'APM index needs converted to ECS format', + url: 'https://www.elastic.co/guide/en/apm/server/master/breaking-changes.html', + details: 'This index was created prior to 7.0.', + reindex: true, + index, + }); + } + + return deprecations; + }, []); +} + +export const isLegacyApmIndex = ( + indexName: string, + apmIndexPatterns: string[] = [], + mappings: FlatSettings['mappings'] +) => { + const clientVersion = get(mappings, '_meta.version', '0.0.0'); + + const find = apmIndexPatterns.find(pattern => { + return minimatch(indexName, pattern) && semver.lt(clientVersion, pkg.version); // no client version or version < 7.0 + }); + + return Boolean(find); +}; + +// source: https://github.com/elastic/apm-integration-testing/blob/master/tests/server/test_upgrade.py +export const apmReindexScript = ` + // add ecs version + ctx._source.ecs = ['version': '1.0.0-beta2']; + + // beat -> observer + def beat = ctx._source.remove("beat"); + if (beat != null) { + beat.remove("name"); + ctx._source.observer = beat; + ctx._source.observer.type = "apm-server"; + } + + def listening = ctx._source.remove("listening"); + if (listening != null) { + ctx._source.observer.listening = listening; + } + + // remove host[.name] + // clarify if we can simply delete this or it will be set somewhere else in 7.0 + ctx._source.remove("host"); + + // docker.container -> container + def docker = ctx._source.remove("docker"); + if (docker != null && docker.containsKey("container")) { + ctx._source.container = docker.container; + } + + // rip up context + HashMap context = ctx._source.remove("context"); + if (context != null) { + // context.process -> process + if (context.containsKey("process")) { + ctx._source.process = context.remove("process"); + ctx._source.process.args = ctx._source.process.remove("argv"); + } + + // context.response -> http.response + HashMap resp = context.remove("response"); + if (resp != null) { + if (! ctx._source.containsKey("http")) { + ctx._source.http = new HashMap(); + } + ctx._source.http.response = resp; + } + + // context.request -> http & url + HashMap request = context.remove("request"); + if (request != null) { + if (! ctx._source.containsKey("http")) { + ctx._source.http = new HashMap(); + } + if (! ctx._source.http.containsKey("request")) { + ctx._source.http.request = new HashMap(); + } + + // context.request.http_version -> http.version + def http_version = request.remove("http_version"); + if (http_version != null) { + ctx._source.http.version = http_version; + } + + // context.request.method -> http.request.method + def method = request.remove("method"); + if (method != null) { + ctx._source.http.request.method = method.toLowerCase(); + } + // context.request.env -> http.request.env + ctx._source.http.request.env = request.remove("env"); + // context.request.socket -> http.request.socket + ctx._source.http.request.socket = request.remove("socket"); + + // context.request.body -> http.request.body.original + def body = request.remove("body"); + if (body != null) { + ctx._source.http.request.body = new HashMap() + //ctx._source.http.request.body.original = body; + // TODO: figure out how to handle body - it can be a string or an object + //request.body = bodyContent; + } + + def parsed = request.remove("cookies"); + if (parsed != null) { + ctx._source.http.request.headers = new HashMap(); + ctx._source.http.request.headers.cookies.parsed = parsed; + } + + if (request.headers.containsKey("cookies") && request.headers.cookies.containsKey("original")) { + if (ctx._source.http.request.headers == null) { + ctx._source.http.request.headers = new HashMap(); + } + ctx._source.http.request.headers.cookies = new HashMap(); + ctx._source.http.request.headers.cookies.original = request.headers.cookies.remove("original"); + } + def ua = request.headers.remove("user-agent"); + if (ua != null) { + if (ctx._source.http.request.headers == null) { + ctx._source.http.request.headers = new HashMap(); + } + //TODO: figure out why user-agent throws an exception + //ctx._source.http.request.headers.user_agent = parsed; + } + def ct = request.headers.remove("content-type"); + if (ct != null) { + if (ctx._source.http.request.headers == null) { + ctx._source.http.request.headers = new HashMap(); + } + //TODO: figure out why content-type throws an exception + ctx._source.http.request.headers.content_type = ct; + } + + // context.request.url -> url + HashMap url = request.remove("url"); + def fragment = url.remove("hash"); + if (fragment != null) { + url.fragment = fragment; + } + def domain = url.remove("hostname"); + if (domain != null) { + url.domain = domain; + } + def path = url.remove("pathname"); + if (path != null) { + url.path = path; + } + def scheme = url.remove("protocol"); + if (scheme != null) { + def end = scheme.lastIndexOf(":"); + if (end > -1) { + scheme = scheme.substring(0, end); + } + url.scheme = scheme + } + def original = url.remove("raw"); + if (original != null) { + url.original = original; + } + def port = url.remove("port"); + if (port != null) { + try { + int portNum = Integer.parseInt(port); + url.port = portNum; + } catch (Exception e) { + // toss port + } + } + def query = url.remove("search"); + if (query != null) { + url.query = query; + } + ctx._source.url = url; + + // restore what is left of request, under http + ctx._source.http.request = request; + } + + // context.service.agent -> agent + HashMap service = context.remove("service"); + ctx._source.agent = service.remove("agent"); + + // context.service -> service + ctx._source.service = service; + + // context.system -> host + def system = context.remove("system"); + if (system != null) { + system.os = new HashMap(); + system.os.platform = system.remove("platform"); + ctx._source.host = system; + } + + // context.tags -> labels + def tags = context.remove("tags"); + if (tags != null) { + ctx._source.labels = tags; + } + + // context.user -> user & user_agent + if (context.containsKey("user")) { + HashMap user = context.remove("user"); + // user.username -> user.name + def username = user.remove("username"); + if (username != null) { + user.name = username; + } + // context.user.ip -> client.ip + if (user.containsKey("ip")) { + ctx._source.client = new HashMap(); + ctx._source.client.ip = user.remove("ip"); + } + // context.user.user-agent -> user_agent.original.text + // XXX: untested + //if (user.containsKey("user-agent")) { + // if (ctx._source.user_agent == null) { + // ctx._source.user_agent = new HashMap(); + // } + // ctx._source.user_agent.original.text = user.remove("user-agent"); + //} + + //TODO: what about user_agent pipelines? + + ctx._source.user = user; + } + + // context.custom -> event.custom + def custom = context.remove("custom"); + if (custom != null) { + if (! ctx._source.containsKey("event")) { + ctx._source.event = new HashMap(); + } + ctx._source.event.custom = custom; + } + + // context.db -> span.db + def db = context.remove("db"); + if (db != null) { + ctx._source.span.db = db; + } + + // context.http -> span.http + def http = context.remove("http"); + if (http != null) { + // context.http.url -> span.http.url.original + def url = http.remove("url"); + if (url != null) { + http.url = ["original": url]; + } + // context.http.status_code -> span.http.response.status_code + def status_code = http.remove("status_code"); + if (status_code != null) { + http.response = ["status_code": status_code]; + } + ctx._source.span.http = http; + } + } + + if (ctx._source.processor.event == "span") { + // bump timestamp.us by span.start.us for spans + // shouldn't @timestamp this already be a Date? + def ts = ctx._source.get("@timestamp"); + if (ts != null && !ctx._source.containsKey("timestamp")) { + // add span.start to @timestamp for rum documents v1 + if (ctx._source.context.service.agent.name == "js-base" && ctx._source.span.start.containsKey("us")) { + ts += ctx._source.span.start.us/1000; + } + } + if (ctx._source.span.containsKey("hex_id")) { + ctx._source.span.id = ctx._source.span.remove("hex_id"); + } + def parent = ctx._source.span.remove("parent"); + if (parent != null && ctx._source.parent == null) { + ctx._source.parent = ["id": parent]; + } + } + + // create trace.id + if (ctx._source.processor.event == "transaction" || ctx._source.processor.event == "span" || ctx._source.processor.event == "error") { + if (ctx._source.containsKey("transaction")) { + def tr_id = ctx._source.transaction.get("id"); + if (ctx._source.trace == null && tr_id != null) { + // create a trace id from the transaction.id + // v1 transaction.id was a UUID, should have 122 random bits or so + ctx._source.trace = new HashMap(); + ctx._source.trace.id = tr_id.replace("-", ""); + } + } + + // create timestamp.us from @timestamp + def ts = ctx._source.get("@timestamp"); + if (ts != null && !ctx._source.containsKey("timestamp")) { + //set timestamp.microseconds to @timestamp + ctx._source.timestamp = new HashMap(); + ctx._source.timestamp.us = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(ts).getTime()*1000; + } + + } + + // transaction.span_count.dropped.total -> transaction.span_count.dropped + if (ctx._source.processor.event == "transaction") { + // transaction.span_count.dropped.total -> transaction.span_count.dropped + if (ctx._source.transaction.containsKey("span_count")) { + def dropped = ctx._source.transaction.span_count.remove("dropped"); + if (dropped != null) { + ctx._source.transaction.span_count.dropped = dropped.total; + } + } + } + + // error.exception is now a list (exception chain) + if (ctx._source.processor.event == "error") { + def exception = ctx._source.error.remove("exception"); + if (exception != null) { + ctx._source.error.exception = [exception]; + } + }`; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json b/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json new file mode 100644 index 0000000000000..cbd361be894e2 --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json @@ -0,0 +1,1600 @@ +{ + "_meta": { + "version": "7.0.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "labels.*" + } + }, + { + "labels": { + "mapping": { + "type": "boolean" + }, + "match_mapping_type": "boolean", + "path_match": "labels.*" + } + }, + { + "labels": { + "mapping": { + "scaling_factor": 1000000, + "type": "scaled_float" + }, + "match_mapping_type": "*", + "path_match": "labels.*" + } + }, + { + "transaction.marks": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "transaction.marks.*" + } + }, + { + "transaction.marks.*.*": { + "mapping": { + "scaling_factor": 1000000, + "type": "scaled_float" + }, + "match_mapping_type": "*", + "path_match": "transaction.marks.*.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "labels": { + "type": "object" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "dynamic": false, + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "culprit": { + "norms": false, + "type": "text" + }, + "exception": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "handled": { + "type": "boolean" + }, + "message": { + "norms": false, + "type": "text" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "grouping_key": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "param_message": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error id icon": { + "ignore_above": 1024, + "type": "keyword" + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "target_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "finished": { + "type": "boolean" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "dynamic": true, + "type": "object" + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "observer": { + "properties": { + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "listening": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "parent": { + "dynamic": false, + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + }, + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "processor": { + "properties": { + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "service": { + "properties": { + "environment": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "framework": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "language": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "sourcemap": { + "dynamic": false, + "properties": { + "bundle_filepath": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "span": { + "dynamic": false, + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "properties": { + "us": { + "type": "long" + } + } + }, + "subtype": { + "ignore_above": 1024, + "type": "keyword" + }, + "sync": { + "type": "boolean" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "cpu": { + "properties": { + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + } + } + }, + "memory": { + "properties": { + "actual": { + "properties": { + "free": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "process": { + "properties": { + "cpu": { + "properties": { + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "size": { + "type": "long" + } + } + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "properties": { + "us": { + "type": "long" + } + } + }, + "trace": { + "dynamic": false, + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "dynamic": false, + "properties": { + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "marks": { + "dynamic": true, + "properties": { + "*": { + "properties": { + "*": { + "dynamic": true, + "type": "object" + } + } + } + }, + "type": "object" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "norms": false, + "type": "text" + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "sampled": { + "type": "boolean" + }, + "span_count": { + "properties": { + "dropped": { + "type": "long" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "view errors": { + "ignore_above": 1024, + "type": "keyword" + }, + "view spans": { + "ignore_above": 1024, + "type": "keyword" + } + } +} diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts index 5fd06feed9419..32b9a5355a5a8 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts @@ -26,7 +26,7 @@ describe('getUpgradeAssistantStatus', () => { }); it('calls /_migration/deprecations', async () => { - await getUpgradeAssistantStatus(callWithRequest, {} as any, '/'); + await getUpgradeAssistantStatus(callWithRequest, {} as any, '/', []); expect(callWithRequest).toHaveBeenCalledWith({}, 'transport.request', { path: '/_migration/deprecations', method: 'GET', @@ -34,7 +34,7 @@ describe('getUpgradeAssistantStatus', () => { }); it('returns the correct shape of data', async () => { - const resp = await getUpgradeAssistantStatus(callWithRequest, {} as any, '/'); + const resp = await getUpgradeAssistantStatus(callWithRequest, {} as any, '/', []); expect(resp).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts index 0682e6acbce9e..3740fd9183e21 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts @@ -7,47 +7,70 @@ import _ from 'lodash'; import { Request } from 'hapi'; -import { DeprecationAPIResponse, DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch'; +import { + CallClusterWithRequest, + DeprecationAPIResponse, + DeprecationInfo, + IndexSettingsDeprecationInfo, +} from 'src/legacy/core_plugins/elasticsearch'; +import { getDeprecatedApmIndices } from './apm'; export interface EnrichedDeprecationInfo extends DeprecationInfo { index?: string; node?: string; + reindex?: boolean; } export interface UpgradeAssistantStatus { cluster: EnrichedDeprecationInfo[]; indices: EnrichedDeprecationInfo[]; - - [checkupType: string]: EnrichedDeprecationInfo[]; } export async function getUpgradeAssistantStatus( - callWithRequest: any, - req: Request, - basePath: string + callWithRequest: CallClusterWithRequest, + request: Request, + basePath: string, + apmIndices: string[] ): Promise { - const deprecations = (await callWithRequest(req, 'transport.request', { - path: '/_migration/deprecations', - method: 'GET', - })) as DeprecationAPIResponse; + const [deprecations, apmIndexDeprecations] = await Promise.all([ + (await callWithRequest(request, 'transport.request', { + path: '/_migration/deprecations', + method: 'GET', + })) as DeprecationAPIResponse, + getDeprecatedApmIndices(callWithRequest, request, apmIndices), + ]); return { cluster: deprecations.cluster_settings.concat(deprecations.node_settings), - indices: getCombinedIndexInfos(deprecations, basePath), + indices: getCombinedIndexInfos(deprecations.index_settings, basePath, apmIndexDeprecations), }; } -// Combines the information from the migration assistance api and the deprecation api into a single array. -// Enhances with information about which index the deprecation applies to and adds buttons for accessing the -// reindex UI. -const getCombinedIndexInfos = (deprecations: DeprecationAPIResponse, basePath: string) => - Object.keys(deprecations.index_settings).reduce( - (indexDeprecations, indexName) => { - return indexDeprecations.concat( - deprecations.index_settings[indexName].map( - d => ({ ...d, index: indexName } as EnrichedDeprecationInfo) - ) - ); - }, - [] as EnrichedDeprecationInfo[] - ); +// Combines the information from the migration assistance API and the required APM indices for re-index +const getCombinedIndexInfos = ( + indexSettings: IndexSettingsDeprecationInfo, + basePath: string, + apmIndexDeprecations: EnrichedDeprecationInfo[] +): EnrichedDeprecationInfo[] => { + const apmIndices = apmIndexDeprecations.reduce((acc, dep) => acc.add(dep.index), new Set()); + + return Object.keys(indexSettings) + .reduce( + (indexDeprecations, indexName) => { + // prevent APM indices from showing up for general re-indexing + if (apmIndices.has(indexName)) { + return indexDeprecations; + } + + return indexDeprecations.concat( + indexSettings[indexName].map(d => ({ + ...d, + index: indexName, + reindex: /Index created before/.test(d.message), + })) + ); + }, + [] as EnrichedDeprecationInfo[] + ) + .concat(apmIndexDeprecations); +}; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts index 91e5e5f1cff7d..ece97c7dee850 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts @@ -8,6 +8,8 @@ import { flow, omit } from 'lodash'; import { ReindexWarning } from '../../../common/types'; import { FlatSettings } from './types'; +import { isLegacyApmIndex } from '../apm'; + /** * Validates, and updates deprecated settings and mappings to be applied to the * new updated index. @@ -23,10 +25,16 @@ export const transformFlatSettings = (flatSettings: FlatSettings) => { * Returns an array of warnings that should be displayed to user before reindexing begins. * @param flatSettings */ -export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[] => { - const warnings = [ - // No warnings yet for 7.0 -> 8.0 - ] as Array<[ReindexWarning, boolean]>; +export const getReindexWarnings = ( + flatSettings: FlatSettings, + apmIndexPatterns: string[] = [] +): ReindexWarning[] => { + const indexName = flatSettings.settings['index.provided_name']; + const apmReindexWarning = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings); + + const warnings = [[ReindexWarning.apmReindex, apmReindexWarning]] as Array< + [ReindexWarning, boolean] + >; return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning); }; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index ce285ff22433b..ed400c80bcba2 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -13,6 +13,8 @@ import { ReindexStep, ReindexWarning, } from '../../../common/types'; +import { apmReindexScript, isLegacyApmIndex } from '../apm'; +import apmMappings from '../apm/mapping.json'; import { getReindexWarnings, transformFlatSettings } from './index_settings'; import { ReindexActions } from './reindex_actions'; @@ -67,7 +69,8 @@ export interface ReindexService { export const reindexServiceFactory = ( callCluster: CallCluster, - actions: ReindexActions + actions: ReindexActions, + apmIndexPatterns: string[] = [] ): ReindexService => { // ------ Utility functions @@ -227,11 +230,13 @@ export const reindexServiceFactory = ( } const { settings, mappings } = transformFlatSettings(flatSettings); + const legacyApmIndex = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings); + const createIndex = await callCluster('indices.create', { index: newIndexName, body: { settings, - mappings, + mappings: legacyApmIndex ? apmMappings : mappings, }, }); @@ -250,6 +255,25 @@ export const reindexServiceFactory = ( */ const startReindexing = async (reindexOp: ReindexSavedObject) => { const { indexName } = reindexOp.attributes; + + const reindexBody = { + source: { index: indexName }, + dest: { index: reindexOp.attributes.newIndexName }, + } as any; + + const flatSettings = await actions.getFlatSettings(indexName); + if (!flatSettings) { + throw Boom.notFound(`Index ${indexName} does not exist.`); + } + + const legacyApmIndex = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings); + if (legacyApmIndex) { + reindexBody.script = { + lang: 'painless', + source: apmReindexScript, + }; + } + const startReindex = (await callCluster('reindex', { refresh: true, waitForCompletion: false, @@ -374,7 +398,7 @@ export const reindexServiceFactory = ( if (!flatSettings) { return null; } else { - return getReindexWarnings(flatSettings); + return getReindexWarnings(flatSettings, apmIndexPatterns); } }, diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts index 72243de7c013c..077400e7567fe 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts @@ -13,11 +13,16 @@ export interface MappingProperties { [key: string]: Mapping; } +interface MetaProperties { + [key: string]: string; +} + export interface FlatSettings { settings: { [key: string]: string; }; mappings: { properties?: MappingProperties; + _meta?: MetaProperties; }; } diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/worker.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/worker.ts index 960704d15db68..583cb738549b0 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/worker.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/worker.ts @@ -45,15 +45,19 @@ export class ReindexWorker { private credentialStore: CredentialStore, private callWithRequest: CallClusterWithRequest, private callWithInternalUser: CallCluster, - private readonly log: Server['log'] + private readonly log: Server['log'], + private apmIndexPatterns: string[] ) { if (ReindexWorker.workerSingleton) { throw new Error(`More than one ReindexWorker cannot be created.`); } + this.apmIndexPatterns = apmIndexPatterns; + this.reindexService = reindexServiceFactory( this.callWithInternalUser, - reindexActionsFactory(this.client, this.callWithInternalUser) + reindexActionsFactory(this.client, this.callWithInternalUser), + apmIndexPatterns ); ReindexWorker.workerSingleton = this; @@ -148,7 +152,7 @@ export class ReindexWorker { const fakeRequest = { headers: credential } as Request; const callCluster = this.callWithRequest.bind(null, fakeRequest) as CallCluster; const actions = reindexActionsFactory(this.client, callCluster); - const service = reindexServiceFactory(callCluster, actions); + const service = reindexServiceFactory(callCluster, actions, this.apmIndexPatterns); reindexOp = await swallowExceptions(service.processNextStep, this.log)(reindexOp); // Update credential store with most recent state. diff --git a/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.ts b/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.ts index cb776793b288a..3d6c631aba455 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.ts @@ -18,7 +18,14 @@ export function registerClusterCheckupRoutes(server: Legacy.Server) { method: 'GET', async handler(request) { try { - return await getUpgradeAssistantStatus(callWithRequest, request, basePath); + const apmIndexPatterns = server.plugins.apm_oss.indexPatterns; + + return await getUpgradeAssistantStatus( + callWithRequest, + request, + basePath, + apmIndexPatterns + ); } catch (e) { if (e.status === 403) { return Boom.forbidden(e.message); diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.ts index 5acc75f8a1e0c..5d333c3cd6511 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.ts @@ -38,7 +38,8 @@ export function registerReindexWorker(server: Server, credentialStore: Credentia credentialStore, callWithRequest, callWithInternalUser, - log + log, + server.plugins.apm_oss.indexPatterns ); // Wait for ES connection before starting the polling loop. @@ -68,7 +69,11 @@ export function registerReindexIndicesRoutes( const callCluster = callWithRequest.bind(null, request) as CallCluster; const reindexActions = reindexActionsFactory(client, callCluster); - const reindexService = reindexServiceFactory(callCluster, reindexActions); + const reindexService = reindexServiceFactory( + callCluster, + reindexActions, + server.plugins.apm_oss.indexPatterns + ); try { const existingOp = await reindexService.findReindexOperation(indexName); @@ -105,7 +110,11 @@ export function registerReindexIndicesRoutes( const { indexName } = request.params; const callCluster = callWithRequest.bind(null, request) as CallCluster; const reindexActions = reindexActionsFactory(client, callCluster); - const reindexService = reindexServiceFactory(callCluster, reindexActions); + const reindexService = reindexServiceFactory( + callCluster, + reindexActions, + server.plugins.apm_oss.indexPatterns + ); try { const reindexOp = await reindexService.findReindexOperation(indexName); From ede668cd2c1fab98153238ffc0a2e17f27df6bb7 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Fri, 1 Feb 2019 22:42:53 -0800 Subject: [PATCH 2/7] Fix types Signed-off-by: Tyler Smalley --- .../components/tabs/checkup/deprecations/index_table.tsx | 2 +- .../public/components/tabs/overview/steps.tsx | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx index d06647038e05f..e2b94c8caff77 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx @@ -15,7 +15,7 @@ const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000]; export interface IndexDeprecationDetails { index: string; - reindex: boolean; + reindex?: boolean; details?: string; } diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/overview/steps.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/overview/steps.tsx index 861db5a74c7a6..13d267cc5c31e 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/overview/steps.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/overview/steps.tsx @@ -95,9 +95,10 @@ const START_UPGRADE_STEP = { export const StepsUI: StatelessComponent< UpgradeAssistantTabProps & ReactIntl.InjectedIntlProps > = ({ checkupData, setSelectedTabIndex, intl }) => { - const countByType = Object.keys(checkupData!).reduce( + const checkupDataTyped = (checkupData! as unknown) as { [checkupType: string]: any[] }; + const countByType = Object.keys(checkupDataTyped).reduce( (counts, checkupType) => { - counts[checkupType] = checkupData![checkupType].length; + counts[checkupType] = checkupDataTyped[checkupType].length; return counts; }, {} as { [checkupType: string]: number } From 28a4bd58c6263583fdb334a3225585fd2a5f7b90 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Sat, 2 Feb 2019 09:37:40 -0800 Subject: [PATCH 3/7] Fixes Signed-off-by: Tyler Smalley --- .../tabs/checkup/deprecations/index_table.tsx | 13 ++++--------- .../tabs/checkup/deprecations/list.tsx | 19 ++++++------------- .../reindex/flyout/warnings_step.tsx | 3 +-- .../server/lib/apm/index.test.ts | 2 +- .../server/lib/reindexing/reindex_service.ts | 5 +---- 5 files changed, 13 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx index e2b94c8caff77..ef3dab848c103 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx @@ -9,18 +9,13 @@ import React from 'react'; import { EuiBasicTable } from '@elastic/eui'; import { injectI18n } from '@kbn/i18n/react'; +import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis'; import { ReindexButton } from './reindex'; const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000]; -export interface IndexDeprecationDetails { - index: string; - reindex?: boolean; - details?: string; -} - export interface IndexDeprecationTableProps extends ReactIntl.InjectedIntlProps { - indices: IndexDeprecationDetails[]; + indices: EnrichedDeprecationInfo[]; } interface IndexDeprecationTableState { @@ -142,8 +137,8 @@ export class IndexDeprecationTableUI extends React.Component< return { actions: [ { - render(indexDep: IndexDeprecationDetails) { - return ; + render(indexDep: EnrichedDeprecationInfo) { + return ; }, }, ], diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx index db838ce137c6f..1c758f62dfe2d 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx @@ -12,7 +12,7 @@ import { GroupByOption } from '../../../types'; import { COLOR_MAP, LEVEL_MAP } from '../constants'; import { DeprecationCell } from './cell'; -import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table'; +import { IndexDeprecationTable } from './index_table'; const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => { return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]); @@ -57,17 +57,16 @@ const SimpleMessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprec }; interface IndexDeprecationProps { - deprecation: DeprecationInfo; - indices: IndexDeprecationDetails[]; + deprecations: EnrichedDeprecationInfo[]; } /** * Shows a single deprecation and table of affected indices with details for each index. */ -const IndexDeprecation: StatelessComponent = ({ deprecation, indices }) => { +const IndexDeprecation: StatelessComponent = ({ deprecations }) => { return ( - - + + ); }; @@ -85,13 +84,7 @@ export const DeprecationList: StatelessComponent<{ if (currentGroupBy === GroupByOption.message && deprecations[0].index !== undefined) { // We assume that every deprecation message is the same issue (since they have the same // message) and that each deprecation will have an index associated with it. - const indices = deprecations.map(dep => ({ - index: dep.index!, - details: dep.details, - reindex: dep.reindex, - })); - - return ; + return ; } else if (currentGroupBy === GroupByOption.index) { return (
diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx index ff45ce1596add..0214cd0fc7c01 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx @@ -117,8 +117,7 @@ export class WarningsFlyoutStep extends React.Component< />

Starting in version 7.0.0, APM data will be represented in the Elastic Common - Schema. In order for legacy data to be included, it is required that the data is - re-indexed to support this new format. + Schema. Historical APM data will not visible until it's reindexed.
{ }); }); -describe.only('isLegacyApmIndex', () => { +describe('isLegacyApmIndex', () => { it('is true when for no version', () => { expect(isLegacyApmIndex('foo-1', ['foo-*'], {})).toEqual(true); }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index ed400c80bcba2..ecd7ad379a299 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -277,10 +277,7 @@ export const reindexServiceFactory = ( const startReindex = (await callCluster('reindex', { refresh: true, waitForCompletion: false, - body: { - source: { index: indexName }, - dest: { index: reindexOp.attributes.newIndexName }, - }, + body: reindexBody, })) as any; return actions.updateReindexOp(reindexOp, { From d4f52ecf6efdbdb0f4648982b5434a3d2287e5f7 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Sat, 2 Feb 2019 23:34:09 -0800 Subject: [PATCH 4/7] Fix tests Signed-off-by: Tyler Smalley --- .../tabs/checkup/deprecations/index_table.tsx | 6 + .../tabs/checkup/deprecations/list.tsx | 18 +- .../__snapshots__/warning_step.test.tsx.snap | 6 +- .../reindex/flyout/warnings_step.tsx | 162 +++++++++--------- .../es_migration_apis.test.ts.snap | 6 + .../server/lib/es_migration_apis.test.ts | 2 + .../lib/reindexing/reindex_service.test.ts | 92 +++++++++- .../server/routes/cluster_checkup.test.ts | 3 + .../server/routes/reindex_indices.test.ts | 3 + 9 files changed, 210 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx index ef3dab848c103..52cfa3732191f 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx @@ -14,6 +14,12 @@ import { ReindexButton } from './reindex'; const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000]; +export interface IndexDeprecationDetails { + index: string; + reindex: boolean; + details?: string; +} + export interface IndexDeprecationTableProps extends ReactIntl.InjectedIntlProps { indices: EnrichedDeprecationInfo[]; } diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx index 1c758f62dfe2d..cb38b848b3bd7 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx @@ -12,7 +12,7 @@ import { GroupByOption } from '../../../types'; import { COLOR_MAP, LEVEL_MAP } from '../constants'; import { DeprecationCell } from './cell'; -import { IndexDeprecationTable } from './index_table'; +import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table'; const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => { return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]); @@ -57,16 +57,17 @@ const SimpleMessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprec }; interface IndexDeprecationProps { - deprecations: EnrichedDeprecationInfo[]; + deprecation: DeprecationInfo; + indices: IndexDeprecationDetails[]; } /** * Shows a single deprecation and table of affected indices with details for each index. */ -const IndexDeprecation: StatelessComponent = ({ deprecations }) => { +const IndexDeprecation: StatelessComponent = ({ deprecation, indices }) => { return ( - - + + ); }; @@ -84,7 +85,12 @@ export const DeprecationList: StatelessComponent<{ if (currentGroupBy === GroupByOption.message && deprecations[0].index !== undefined) { // We assume that every deprecation message is the same issue (since they have the same // message) and that each deprecation will have an index associated with it. - return ; + const indices = deprecations.map(dep => ({ + index: dep.index!, + details: dep.details, + reindex: dep.reindex === true, + })); + return ; } else if (currentGroupBy === GroupByOption.index) { return (

diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap index bd06a5e6b82d3..c1e7e58813425 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap @@ -108,8 +108,7 @@ exports[`WarningsFlyoutStep renders 1`] = ` 1 - ), reindexing converts these fields to - + ), reindexing converts these fields to true @@ -129,6 +128,9 @@ exports[`WarningsFlyoutStep renders 1`] = `

+ {warnings.includes(ReindexWarning.allField) && ( - - - _all field will be removed - - } - checked={checkedIds[idForWarning(ReindexWarning.allField)]} - onChange={this.onChange} - /> -

- The _all meta field is no longer supported in 7.0. Reindexing - removes the _all field in the new index. Ensure that no - application code or scripts reply on this field. -
- - Documentation - -

- + + + + _all field will be removed + + } + checked={checkedIds[idForWarning(ReindexWarning.allField)]} + onChange={this.onChange} + /> +

+ The _all meta field is no longer supported in 7.0. Reindexing + removes the _all field in the new index. Ensure that no + application code or scripts reply on this field. +
+ + Documentation + +

+ + + + )} - - {warnings.includes(ReindexWarning.apmReindex) && ( - - This index will be converted to ECS format} - checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]} - onChange={this.onChange} - /> -

- Starting in version 7.0.0, APM data will be represented in the Elastic Common - Schema. Historical APM data will not visible until it's reindexed. -
- - Documentation - -
- - More about ECS - -

- + + + This index will be converted to ECS format} + checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]} + onChange={this.onChange} + /> +

+ Starting in version 7.0.0, APM data will be represented in the Elastic Common + Schema. Historical APM data will not visible until it's reindexed. +
+ + Documentation + +
+ + More about ECS + +

+ + + + )} - - {warnings.includes(ReindexWarning.booleanFields) && ( - - - Boolean data in _source might change - - } - checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]} - onChange={this.onChange} - /> -

- If a documents contain a boolean field that is neither true or{' '} - false (for example, "yes",{' '} - "on", 1), reindexing converts these fields to{' '} - true or false. Ensure that no application code - or scripts rely on boolean fields in the deprecated format. -
- - Documentation - -

- + + + + Boolean data in _source might change + + } + checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]} + onChange={this.onChange} + /> +

+ If a documents contain a boolean field that is neither true or{' '} + false (for example, "yes",{' '} + "on", 1), reindexing converts these fields + to true or false. Ensure that no application + code or scripts rely on boolean fields in the deprecated format. +
+ + Documentation + +

+ + + + )} diff --git a/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap b/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap index 882911a44bfaf..c5038f107ac26 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap +++ b/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap @@ -28,6 +28,7 @@ Object { "index": ".monitoring-es-6-2018.11.07", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { @@ -35,6 +36,7 @@ Object { "index": "twitter", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { @@ -42,6 +44,7 @@ Object { "index": ".kibana", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { @@ -49,6 +52,7 @@ Object { "index": ".watcher-history-6-2018.11.07", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { @@ -56,6 +60,7 @@ Object { "index": ".monitoring-kibana-6-2018.11.07", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { @@ -63,6 +68,7 @@ Object { "index": "twitter2", "level": "warning", "message": "Coercion of boolean fields", + "reindex": false, "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, ], diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts index 32b9a5355a5a8..fa33ffe87bdb1 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts @@ -16,6 +16,8 @@ describe('getUpgradeAssistantStatus', () => { const callWithRequest = jest.fn().mockImplementation(async (req, api, { path }) => { if (path === '/_migration/deprecations') { return deprecationsResponse; + } else if (api === 'indices.getMapping') { + return {}; } else { throw new Error(`Unexpected API call: ${path}`); } diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 5f299070b1171..7c90905ea76e6 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -11,6 +11,8 @@ import { ReindexStatus, ReindexStep, } from '../../../common/types'; +import { apmReindexScript } from '../apm'; +import apmMappings from '../apm/mapping.json'; import { ReindexService, reindexServiceFactory } from './reindex_service'; describe('reindexService', () => { @@ -42,19 +44,22 @@ describe('reindexService', () => { runWhileMlLocked: jest.fn(async (f: any) => f({ attributes: {} })), }; callCluster = jest.fn(); - service = reindexServiceFactory(callCluster, actions); + service = reindexServiceFactory(callCluster, actions, ['apm-*']); }); describe('detectReindexWarnings', () => { it('fetches reindex warnings from flat settings', async () => { + const indexName = 'myIndex'; actions.getFlatSettings.mockResolvedValueOnce({ - settings: {}, + settings: { + 'index.provided_name': indexName, + }, mappings: { properties: { https: { type: 'boolean' } }, }, }); - const reindexWarnings = await service.detectReindexWarnings('myIndex'); + const reindexWarnings = await service.detectReindexWarnings(indexName); expect(reindexWarnings).toEqual([]); }); @@ -461,6 +466,43 @@ describe('reindexService', () => { }); }); + it('used APM mapping for legacy APM index', async () => { + const indexName = 'apm-1'; + const newIndexName = 'apm-1-reindexed'; + + actions.getFlatSettings.mockResolvedValueOnce({ + settings: { + 'index.number_of_replicas': 5, + }, + mappings: { + _meta: { + version: '6.7.0', + }, + }, + }); + + callCluster.mockResolvedValueOnce({ acknowledged: true }); // indices.create + await service.processNextStep({ + id: '1', + attributes: { + ...defaultAttributes, + indexName, + newIndexName, + lastCompletedStep: ReindexStep.readonly, + }, + } as ReindexSavedObject); + + expect(callCluster).toHaveBeenCalledWith('indices.create', { + index: newIndexName, + body: { + mappings: apmMappings, + settings: { + 'index.number_of_replicas': 5, + }, + }, + }); + }); + it('fails if create index is not acknowledged', async () => { callCluster .mockResolvedValueOnce({ myIndex: settingsMappings }) @@ -495,6 +537,13 @@ describe('reindexService', () => { attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.newIndexCreated }, } as ReindexSavedObject; + beforeEach(() => { + actions.getFlatSettings.mockResolvedValueOnce({ + settings: {}, + mappings: {}, + }); + }); + it('starts reindex, saves taskId, and updates lastCompletedStep', async () => { callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex const updatedOp = await service.processNextStep(reindexOp); @@ -511,6 +560,43 @@ describe('reindexService', () => { }); }); + it('uses APM script for legacy APM index', async () => { + const indexName = 'apm-1'; + const newIndexName = 'apm-1-reindexed'; + + callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex + actions.getFlatSettings.mockResolvedValueOnce({ + settings: {}, + mappings: { + _meta: { + version: '6.7.0', + }, + }, + }); + + await service.processNextStep({ + id: '1', + attributes: { + ...defaultAttributes, + indexName, + newIndexName, + lastCompletedStep: ReindexStep.newIndexCreated, + }, + } as ReindexSavedObject); + expect(callCluster).toHaveBeenLastCalledWith('reindex', { + refresh: true, + waitForCompletion: false, + body: { + source: { index: indexName }, + dest: { index: newIndexName }, + script: { + lang: 'painless', + source: apmReindexScript, + }, + }, + }); + }); + it('fails if starting reindex fails', async () => { callCluster.mockRejectedValueOnce(new Error('blah!')).mockResolvedValueOnce({}); const updatedOp = await service.processNextStep(reindexOp); diff --git a/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.test.ts index 0aee001e89066..47593e77350f8 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.test.ts @@ -23,6 +23,9 @@ describe('cluster checkup API', () => { elasticsearch: { getCluster: () => ({ callWithRequest: jest.fn() } as any), } as any, + apm_oss: { + indexPatterns: ['apm-*'], + } as any, } as any; server.config = () => ({ get: () => '' } as any); diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.test.ts index 3e88ff5d6888b..2dcf3d7f0bcae 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.test.ts @@ -41,6 +41,9 @@ describe('reindex template API', () => { elasticsearch: { getCluster: () => ({ callWithRequest: jest.fn() } as any), } as any, + apm_oss: { + indexPatterns: ['apm-*'] as string[], + } as any, } as any; server.config = () => ({ get: () => '' } as any); server.decorate('request', 'getSavedObjectsClient', () => jest.fn()); From 3f3126e1549f30c9fbaa58879bb422f0dea27da7 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Sun, 3 Feb 2019 01:05:38 -0800 Subject: [PATCH 5/7] Update types Signed-off-by: Tyler Smalley --- .../components/tabs/checkup/deprecations/index_table.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx index 52cfa3732191f..5a1deb59c270e 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx @@ -9,7 +9,6 @@ import React from 'react'; import { EuiBasicTable } from '@elastic/eui'; import { injectI18n } from '@kbn/i18n/react'; -import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis'; import { ReindexButton } from './reindex'; const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000]; @@ -21,7 +20,7 @@ export interface IndexDeprecationDetails { } export interface IndexDeprecationTableProps extends ReactIntl.InjectedIntlProps { - indices: EnrichedDeprecationInfo[]; + indices: IndexDeprecationDetails[]; } interface IndexDeprecationTableState { @@ -143,7 +142,7 @@ export class IndexDeprecationTableUI extends React.Component< return { actions: [ { - render(indexDep: EnrichedDeprecationInfo) { + render(indexDep: IndexDeprecationDetails) { return ; }, }, From 96f6f58b8432cfab486b8357a785608844b471c5 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Mon, 4 Feb 2019 13:23:03 -0800 Subject: [PATCH 6/7] Updates messaging Signed-off-by: Tyler Smalley --- .../checkup/deprecations/reindex/flyout/warnings_step.tsx | 6 +----- x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx index 812b14c37db26..d14248b90064b 100644 --- a/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx @@ -123,15 +123,11 @@ export class WarningsFlyoutStep extends React.Component< Schema. Historical APM data will not visible until it's reindexed.
Documentation -
- - More about ECS -

diff --git a/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts index 33eb988dd34fc..3c258871771e6 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts @@ -30,9 +30,9 @@ export async function getDeprecatedApmIndices( if (semver.lt(get(indices[index], 'mappings._meta.version', '0.0.0'), pkg.version)) { deprecations.push({ level: 'critical', - message: 'APM index needs converted to ECS format', - url: 'https://www.elastic.co/guide/en/apm/server/master/breaking-changes.html', - details: 'This index was created prior to 7.0.', + message: 'APM index needs converted to 7.x format', + url: 'https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html', + details: 'This index was created prior to 7.0', reindex: true, index, }); From 4d6af8819f7068e9557f28de42d5c9910fc2c393 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 5 Feb 2019 18:49:40 -0800 Subject: [PATCH 7/7] Updates APM reindex script/mapping Signed-off-by: Tyler Smalley --- .../upgrade_assistant/server/lib/apm/index.ts | 139 ++++++++++-------- .../server/lib/apm/mapping.json | 80 +++++----- 2 files changed, 113 insertions(+), 106 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts index 8602da3f75bd4..ee5dd5a0562b1 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts @@ -69,6 +69,13 @@ export const apmReindexScript = ` ctx._source.observer.type = "apm-server"; } + if (! ctx._source.containsKey("observer")) { + ctx._source.observer = new HashMap(); + } + + // observer.major_version + ctx._source.observer.version_major = 7; + def listening = ctx._source.remove("listening"); if (listening != null) { ctx._source.observer.listening = listening; @@ -108,9 +115,6 @@ export const apmReindexScript = ` if (! ctx._source.containsKey("http")) { ctx._source.http = new HashMap(); } - if (! ctx._source.http.containsKey("request")) { - ctx._source.http.request = new HashMap(); - } // context.request.http_version -> http.version def http_version = request.remove("http_version"); @@ -118,54 +122,8 @@ export const apmReindexScript = ` ctx._source.http.version = http_version; } - // context.request.method -> http.request.method - def method = request.remove("method"); - if (method != null) { - ctx._source.http.request.method = method.toLowerCase(); - } - // context.request.env -> http.request.env - ctx._source.http.request.env = request.remove("env"); - // context.request.socket -> http.request.socket - ctx._source.http.request.socket = request.remove("socket"); - - // context.request.body -> http.request.body.original - def body = request.remove("body"); - if (body != null) { - ctx._source.http.request.body = new HashMap() - //ctx._source.http.request.body.original = body; - // TODO: figure out how to handle body - it can be a string or an object - //request.body = bodyContent; - } - - def parsed = request.remove("cookies"); - if (parsed != null) { - ctx._source.http.request.headers = new HashMap(); - ctx._source.http.request.headers.cookies.parsed = parsed; - } + ctx._source.http.request = new HashMap(); - if (request.headers.containsKey("cookies") && request.headers.cookies.containsKey("original")) { - if (ctx._source.http.request.headers == null) { - ctx._source.http.request.headers = new HashMap(); - } - ctx._source.http.request.headers.cookies = new HashMap(); - ctx._source.http.request.headers.cookies.original = request.headers.cookies.remove("original"); - } - def ua = request.headers.remove("user-agent"); - if (ua != null) { - if (ctx._source.http.request.headers == null) { - ctx._source.http.request.headers = new HashMap(); - } - //TODO: figure out why user-agent throws an exception - //ctx._source.http.request.headers.user_agent = parsed; - } - def ct = request.headers.remove("content-type"); - if (ct != null) { - if (ctx._source.http.request.headers == null) { - ctx._source.http.request.headers = new HashMap(); - } - //TODO: figure out why content-type throws an exception - ctx._source.http.request.headers.content_type = ct; - } // context.request.url -> url HashMap url = request.remove("url"); @@ -209,7 +167,18 @@ export const apmReindexScript = ` ctx._source.url = url; // restore what is left of request, under http + + def body = request.remove("body"); + ctx._source.http.request = request; + ctx._source.http.request.method = ctx._source.http.request.method?.toLowerCase(); + + // context.request.body -> http.request.body.original + if (body != null) { + ctx._source.http.request.body = new HashMap(); + ctx._source.http.request.body.original = body; + } + } // context.service.agent -> agent @@ -241,32 +210,65 @@ export const apmReindexScript = ` if (username != null) { user.name = username; } + // context.user.ip -> client.ip if (user.containsKey("ip")) { ctx._source.client = new HashMap(); ctx._source.client.ip = user.remove("ip"); } - // context.user.user-agent -> user_agent.original.text - // XXX: untested - //if (user.containsKey("user-agent")) { - // if (ctx._source.user_agent == null) { - // ctx._source.user_agent = new HashMap(); - // } - // ctx._source.user_agent.original.text = user.remove("user-agent"); - //} - //TODO: what about user_agent pipelines? + def ua = user.remove("user-agent"); + if (ua != null) { + ctx._source.user_agent = new HashMap(); + // setting original and original.text is not possible in painless + // as original is a keyword in ES template we cannot set it to a HashMap here, + // so the following is the only possible solution: + ctx._source.user_agent.original = ua.substring(0, Integer.min(1024, ua.length())); + } + + def pua = user.remove("user_agent"); + if (pua != null) { + if (ctx._source.user_agent == null){ + ctx._source.user_agent = new HashMap(); + } + def os = pua.remove("os"); + def osminor = pua.remove("os_minor"); + def osmajor = pua.remove("os_major"); + def osname = pua.remove("os_name"); + if (osminor != null || osmajor != null || osname != null){ + ctx._source.user_agent.os = new HashMap(); + ctx._source.user_agent.os.full = os; + ctx._source.user_agent.os.version = osmajor + "." + osminor; + ctx._source.user_agent.os.name = osname; + } + + def device = pua.remove("device"); + if (device != null){ + ctx._source.user_agent.device = new HashMap(); + ctx._source.user_agent.device.name = device; + } + // not exactly reflecting 7.0, but the closes we can get + def patch = pua.remove("patch"); + def minor = pua.remove("minor"); + def major = pua.remove("major"); + if (patch != null || minor != null || major != null){ + ctx._source.user_agent.version = major + "." + minor + "." + patch; + } + } ctx._source.user = user; } - // context.custom -> event.custom + // context.custom -> error,transaction,span.custom def custom = context.remove("custom"); if (custom != null) { - if (! ctx._source.containsKey("event")) { - ctx._source.event = new HashMap(); + if (ctx._source.processor.event == "span") { + ctx._source.span.custom = custom; + } else if (ctx._source.processor.event == "transaction") { + ctx._source.transaction.custom = custom; + } else if (ctx._source.processor.event == "error") { + ctx._source.error.custom = custom; } - ctx._source.event.custom = custom; } // context.db -> span.db @@ -344,10 +346,17 @@ export const apmReindexScript = ` } } - // error.exception is now a list (exception chain) if (ctx._source.processor.event == "error") { + // culprit is now a keyword, so trim it down to 1024 chars + def culprit = ctx._source.error.remove("culprit"); + if (culprit != null) { + ctx._source.error.culprit = culprit.substring(0, Integer.min(1024, culprit.length())); + } + + // error.exception is now a list (exception chain) def exception = ctx._source.error.remove("exception"); if (exception != null) { ctx._source.error.exception = [exception]; } - }`; + } +`; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json b/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json index cbd361be894e2..f518b824f1108 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json +++ b/x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json @@ -1,5 +1,6 @@ { "_meta": { + "beat": "apm", "version": "7.0.0" }, "date_detection": false, @@ -93,6 +94,7 @@ "type": "date" }, "agent": { + "dynamic": false, "properties": { "ephemeral_id": { "ignore_above": 1024, @@ -121,6 +123,7 @@ } }, "client": { + "dynamic": false, "properties": { "address": { "ignore_above": 1024, @@ -236,6 +239,7 @@ } }, "container": { + "dynamic": false, "properties": { "id": { "ignore_above": 1024, @@ -356,8 +360,8 @@ "type": "keyword" }, "culprit": { - "norms": false, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "exception": { "properties": { @@ -420,10 +424,6 @@ } } }, - "error id icon": { - "ignore_above": 1024, - "type": "keyword" - }, "event": { "properties": { "action": { @@ -568,6 +568,7 @@ } }, "host": { + "dynamic": false, "properties": { "architecture": { "ignore_above": 1024, @@ -662,6 +663,7 @@ } }, "http": { + "dynamic": false, "properties": { "request": { "properties": { @@ -720,6 +722,7 @@ } }, "kubernetes": { + "dynamic": false, "properties": { "annotations": { "type": "object" @@ -833,6 +836,7 @@ } }, "observer": { + "dynamic": false, "properties": { "geo": { "properties": { @@ -927,6 +931,9 @@ "version": { "ignore_above": 1024, "type": "keyword" + }, + "version_major": { + "type": "byte" } } }, @@ -980,6 +987,7 @@ } }, "process": { + "dynamic": false, "properties": { "args": { "ignore_above": 1024, @@ -1102,6 +1110,7 @@ } }, "service": { + "dynamic": false, "properties": { "environment": { "ignore_above": 1024, @@ -1413,13 +1422,13 @@ }, "name": { "fields": { - "keyword": { - "ignore_above": 1024, - "type": "keyword" + "text": { + "norms": false, + "type": "text" } }, - "norms": false, - "type": "text" + "ignore_above": 1024, + "type": "keyword" }, "result": { "ignore_above": 1024, @@ -1442,6 +1451,7 @@ } }, "url": { + "dynamic": false, "properties": { "domain": { "ignore_above": 1024, @@ -1455,10 +1465,6 @@ "ignore_above": 1024, "type": "keyword" }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, "original": { "ignore_above": 1024, "type": "keyword" @@ -1489,6 +1495,7 @@ } }, "user": { + "dynamic": false, "properties": { "email": { "ignore_above": 1024, @@ -1499,8 +1506,16 @@ "type": "keyword" }, "group": { - "ignore_above": 1024, - "type": "keyword" + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } }, "hash": { "ignore_above": 1024, @@ -1517,18 +1532,15 @@ } }, "user_agent": { + "dynamic": false, "properties": { "device": { - "ignore_above": 1024, - "type": "keyword" - }, - "major": { - "ignore_above": 1024, - "type": "keyword" - }, - "minor": { - "ignore_above": 1024, - "type": "keyword" + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } }, "name": { "ignore_above": 1024, @@ -1558,12 +1570,6 @@ "ignore_above": 1024, "type": "keyword" }, - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, "name": { "ignore_above": 1024, "type": "keyword" @@ -1578,20 +1584,12 @@ } } }, - "patch": { - "ignore_above": 1024, - "type": "keyword" - }, "version": { "ignore_above": 1024, "type": "keyword" } } }, - "view errors": { - "ignore_above": 1024, - "type": "keyword" - }, "view spans": { "ignore_above": 1024, "type": "keyword"