From c5894d3ad78c46cd144ba10e651e5856d720d345 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 19 Dec 2019 15:24:40 -0600 Subject: [PATCH 1/6] Revert pinned ES snapshot Elasticsearch was pinned to an older version due to a migrations failure. Root cause was determined to be upstream and tracked at elastic/elasticsearch#50248. Now that a fix is merged we can see where the most recent snapshot leaves us. --- packages/kbn-es/src/custom_snapshots.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/kbn-es/src/custom_snapshots.js b/packages/kbn-es/src/custom_snapshots.js index 74de3c2c792fd..35673c58d9eef 100644 --- a/packages/kbn-es/src/custom_snapshots.js +++ b/packages/kbn-es/src/custom_snapshots.js @@ -24,9 +24,10 @@ function isVersionFlag(a) { } function getCustomSnapshotUrl() { - // force use of manually created snapshots until ReindexPutMappings fix + // force use of manually created snapshots if (!process.env.KBN_ES_SNAPSHOT_URL && !process.argv.some(isVersionFlag)) { - return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}'; + // return 'https://storage.googleapis.com/kibana-ci-tmp-artifacts/{name}-{version}-{os}-x86_64.{ext}'; + return undefined; } if (process.env.KBN_ES_SNAPSHOT_URL && process.env.KBN_ES_SNAPSHOT_URL !== 'false') { From 13752861a68fe178a66422c769bd1728458a5f24 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 19 Dec 2019 15:28:42 -0600 Subject: [PATCH 2/6] Temporarily create ES snapshot from source Until new artifacts are built we need to build from source to test the mappings fix. This is for testing the most recent fixes and should be reverted before merging. --- src/dev/ci_setup/load_env_keys.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh index 62d29db232eae..00dd854db77e9 100644 --- a/src/dev/ci_setup/load_env_keys.sh +++ b/src/dev/ci_setup/load_env_keys.sh @@ -37,3 +37,6 @@ else # remove vault related secrets unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR fi + +# temporarily test against source until new artifacts are built +export TEST_ES_FROM="source" From cd45d6ffb439a48f3ece7b8695301a06c5ff0738 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 20 Dec 2019 10:15:37 -0600 Subject: [PATCH 3/6] Revert "Temporarily create ES snapshot from source" This reverts commit 13752861a68fe178a66422c769bd1728458a5f24. --- src/dev/ci_setup/load_env_keys.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dev/ci_setup/load_env_keys.sh b/src/dev/ci_setup/load_env_keys.sh index 00dd854db77e9..62d29db232eae 100644 --- a/src/dev/ci_setup/load_env_keys.sh +++ b/src/dev/ci_setup/load_env_keys.sh @@ -37,6 +37,3 @@ else # remove vault related secrets unset VAULT_ROLE_ID VAULT_SECRET_ID VAULT_TOKEN VAULT_ADDR fi - -# temporarily test against source until new artifacts are built -export TEST_ES_FROM="source" From 857dc5e6329afe65114c00a37a221505a02a5a7f Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 20 Dec 2019 13:30:45 -0600 Subject: [PATCH 4/6] Enable field data for _id in .kibana In 8.0 field data is defaulted to false in Elasticsearch. We make use if this in several areas. The setting change is to reduce memory pressure. The Kibana index typically isn't very large so this is not expected to have a large impact. --- .../server/saved_objects/migrations/core/elastic_index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index e7621d88f78ee..ac9f9557d750e 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -34,6 +34,7 @@ export interface FullIndexInfo { exists: boolean; indexName: string; mappings: IndexMapping; + settings: { [name: string]: any }; } /** @@ -51,6 +52,9 @@ export async function fetchInfo(callCluster: CallCluster, index: string): Promis aliases: {}, exists: false, indexName: index, + settings: { + 'indices.id_field_data.enabled': true, + }, mappings: { dynamic: 'strict', properties: {} }, }; } From 0ac7f64be7e8630a3046408527a8cbc0d23642f5 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 20 Dec 2019 15:11:03 -0600 Subject: [PATCH 5/6] pass setings through migration context --- .../server/saved_objects/migrations/core/migration_context.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index d4e97ee6c5747..deb8484fef6d7 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -76,7 +76,6 @@ export async function migrationContext(opts: MigrationOpts): Promise { const alias = opts.index; const source = createSourceContext(await fetchInfo(callCluster, alias), alias); const dest = createDestContext(source, alias, opts.mappingProperties); - return { callCluster, alias, @@ -107,7 +106,7 @@ function createSourceContext(source: FullIndexInfo, alias: string) { function createDestContext( source: FullIndexInfo, alias: string, - mappingProperties: MappingProperties + mappingProperties: MappingProperties, ): FullIndexInfo { const activeMappings = buildActiveMappings({ properties: mappingProperties }); @@ -115,6 +114,7 @@ function createDestContext( aliases: {}, exists: false, indexName: nextIndexName(source.indexName, alias), + settings: source.settings, mappings: { ...activeMappings, properties: { From 2a02f46d3eb34b50a07264144a816b8354441e10 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 20 Dec 2019 15:36:45 -0600 Subject: [PATCH 6/6] lint --- .../server/saved_objects/migrations/core/migration_context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index deb8484fef6d7..002413ba97a26 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -106,7 +106,7 @@ function createSourceContext(source: FullIndexInfo, alias: string) { function createDestContext( source: FullIndexInfo, alias: string, - mappingProperties: MappingProperties, + mappingProperties: MappingProperties ): FullIndexInfo { const activeMappings = buildActiveMappings({ properties: mappingProperties });