From 95060593bc150e76c130d6fa1399b545ee30f45d Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 24 Aug 2020 12:28:32 +0200 Subject: [PATCH 01/11] Move internal index templates to composable templates --- .../SnapshotLifecycleTemplateRegistry.java | 2 +- .../core/src/main/resources/ilm-history.json | 27 +- .../core/src/main/resources/slm-history.json | 30 +- .../src/main/resources/triggered-watches.json | 24 +- .../main/resources/watch-history-no-ilm.json | 384 +++++++++--------- .../src/main/resources/watch-history.json | 342 ++++++++-------- .../core/src/main/resources/watches.json | 110 ++--- ...napshotLifecycleTemplateRegistryTests.java | 8 +- .../history/ILMHistoryTemplateRegistry.java | 2 +- .../support/WatcherIndexTemplateRegistry.java | 2 +- .../WatcherIndexTemplateRegistryTests.java | 30 +- 11 files changed, 486 insertions(+), 475 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index 908d469d0eba8..6ec9a117574ab 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -69,7 +69,7 @@ public SnapshotLifecycleTemplateRegistry(Settings nodeSettings, ClusterService c } @Override - protected List getLegacyTemplateConfigs() { + protected List getComposableTemplateConfigs() { if (slmHistoryEnabled == false) { return Collections.emptyList(); } diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index bb79af74f2034..b640e7eb2ed6a 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -2,18 +2,14 @@ "index_patterns": [ "ilm-history-${xpack.ilm_history.template.version}*" ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.lifecycle.name": "ilm-history-ilm-policy", - "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}", - "index.hidden": true, - "index.format": 1 - }, - "mappings": { - "_doc": { + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "ilm-history-ilm-policy" + }, + "mappings": { "dynamic": false, "properties": { "@timestamp": { @@ -26,7 +22,7 @@ "index": { "type": "keyword" }, - "index_age":{ + "index_age": { "type": "long" }, "success": { @@ -81,5 +77,10 @@ } } }, + "_meta": { + "description": "index template for ILM history indices", + "managed": true + }, + "priority": 2147483647, "version": ${xpack.ilm_history.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/slm-history.json b/x-pack/plugin/core/src/main/resources/slm-history.json index 7ddd995fb21eb..b9320c170ecd4 100644 --- a/x-pack/plugin/core/src/main/resources/slm-history.json +++ b/x-pack/plugin/core/src/main/resources/slm-history.json @@ -2,18 +2,18 @@ "index_patterns": [ ".slm-history-${xpack.slm.template.version}*" ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.lifecycle.name": "slm-history-ilm-policy", - "index.lifecycle.rollover_alias": ".slm-history-${xpack.slm.template.version}", - "index.hidden": true, - "index.format": 1 - }, - "mappings": { - "_doc": { + "priority": 2147483647, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "slm-history-ilm-policy", + "index.lifecycle.rollover_alias": ".slm-history-${xpack.slm.template.version}", + "index.hidden": true, + "index.format": 1 + }, + "mappings": { "dynamic": false, "properties": { "@timestamp": { @@ -26,7 +26,7 @@ "repository": { "type": "keyword" }, - "snapshot_name":{ + "snapshot_name": { "type": "keyword" }, "operation": { @@ -57,5 +57,9 @@ } } }, + "_meta": { + "description": "index template for SLM history indices", + "managed": true + }, "version": ${xpack.slm.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index 28f3b0fea6537..f0a8a4f418292 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -1,21 +1,21 @@ { "index_patterns": [ ".triggered_watches*" ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.auto_expand_replicas": "0-1", - "index.refresh_interval" : "-1", - "index.format": 6, - "index.priority": 900 - }, - "mappings": { - "_doc": { - "dynamic" : "strict", + "priority": 2147483647, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.auto_expand_replicas": "0-1", + "index.refresh_interval": "-1", + "index.format": 6, + "index.priority": 900 + }, + "mappings": { + "dynamic": "strict", "properties": { "trigger_event": { "type": "object", "dynamic": true, - "enabled" : false, + "enabled": false, "properties": { "schedule": { "type": "object", diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 7b5910508392d..185a47b96971b 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -1,15 +1,15 @@ { "index_patterns": [ ".watcher-history-${xpack.watcher.template.version}*" ], - "order": 2147483646, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.hidden": true, - "index.format": 6 - }, - "mappings": { - "doc": { + "priority": 2147483646, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.hidden": true, + "index.format": 6 + }, + "mappings": { "_meta": { "watcher-history-version": "${xpack.watcher.template.version}" }, @@ -46,7 +46,7 @@ }, { "disabled_jira_custom_fields": { - "path_match": "result.actions.jira.fields.customfield_*", + "path_match": "result.actions.jira.fields.customfield_*", "mapping": { "type": "object", "enabled": false @@ -66,8 +66,8 @@ "type": "object", "dynamic": true, "properties": { - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, "triggered_time": { "type": "date" @@ -98,9 +98,9 @@ } } }, - "vars" : { - "type" : "object", - "enabled" : false + "vars": { + "type": "object", + "enabled": false }, "input": { "type": "object", @@ -115,8 +115,8 @@ }, "status": { "type": "object", - "enabled" : false, - "dynamic" : true + "enabled": false, + "dynamic": true }, "messages": { "type": "text" @@ -124,9 +124,9 @@ "user": { "type": "text" }, - "exception" : { - "type" : "object", - "enabled" : false + "exception": { + "type": "object", + "enabled": false }, "result": { "type": "object", @@ -142,15 +142,15 @@ "type": "object", "dynamic": true, "properties": { - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "payload" : { - "type" : "object", - "enabled" : false + "payload": { + "type": "object", + "enabled": false }, "search": { "type": "object", @@ -193,53 +193,53 @@ } } }, - "condition" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "condition": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "met" : { - "type" : "boolean" + "met": { + "type": "boolean" }, - "compare" : { - "type" : "object", - "enabled" : false + "compare": { + "type": "object", + "enabled": false }, - "array_compare" : { - "type" : "object", - "enabled" : false + "array_compare": { + "type": "object", + "enabled": false }, - "script" : { - "type" : "object", - "enabled" : false + "script": { + "type": "object", + "enabled": false } } }, - "transform" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "transform": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "search" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "request" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "indices" : { - "type" : "keyword" - }, - "types" : { - "type" : "keyword" + "search": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "indices": { + "type": "keyword" + }, + "types": { + "type": "keyword" } } } @@ -252,17 +252,17 @@ "include_in_parent": true, "dynamic": true, "properties": { - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" }, - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "reason" : { - "type" : "keyword" + "reason": { + "type": "keyword" }, "email": { "type": "object", @@ -333,7 +333,7 @@ } } }, - "hipchat" : { + "hipchat": { "type": "object", "dynamic": true, "properties": { @@ -351,38 +351,38 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, - "room" : { + "room": { "type": "keyword" }, - "user" : { + "user": { "type": "keyword" }, - "message" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "message_format" : { - "type" : "keyword" + "message": { + "type": "object", + "dynamic": true, + "properties": { + "message_format": { + "type": "keyword" }, - "color" : { - "type" : "keyword" + "color": { + "type": "keyword" }, - "notify" : { - "type" : "boolean" + "notify": { + "type": "boolean" }, - "message" : { - "type" : "text" + "message": { + "type": "text" }, - "from" : { - "type" : "text" + "from": { + "type": "text" } } } @@ -390,7 +390,7 @@ } } }, - "jira" : { + "jira": { "type": "object", "dynamic": true, "properties": { @@ -400,13 +400,13 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, "fields": { "type": "object", @@ -418,30 +418,30 @@ "description": { "type": "text" }, - "labels" : { + "labels": { "type": "text" }, - "project" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "key" : { - "type" : "keyword" + "project": { + "type": "object", + "dynamic": true, + "properties": { + "key": { + "type": "keyword" }, - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" } } }, - "issuetype" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "name" : { + "issuetype": { + "type": "object", + "dynamic": true, + "properties": { + "name": { "type": "keyword" }, - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" } } } @@ -450,21 +450,21 @@ "result": { "type": "object", "dynamic": true, - "properties" : { - "id" : { - "type" : "keyword" + "properties": { + "id": { + "type": "keyword" }, - "key" : { - "type" : "keyword" + "key": { + "type": "keyword" }, - "self" : { - "type" : "keyword" + "self": { + "type": "keyword" } } } } }, - "slack" : { + "slack": { "type": "object", "dynamic": true, "properties": { @@ -482,42 +482,42 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, - "to" : { + "to": { "type": "keyword" }, - "message" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "from" : { - "type" : "text" + "message": { + "type": "object", + "dynamic": true, + "properties": { + "from": { + "type": "text" }, - "icon" : { - "type" : "keyword" + "icon": { + "type": "keyword" }, - "text" : { - "type" : "text" + "text": { + "type": "text" }, - "attachments" : { - "type" : "nested", + "attachments": { + "type": "nested", "include_in_parent": true, - "dynamic" : true, - "properties" : { - "color" : { - "type" : "keyword" + "dynamic": true, + "properties": { + "color": { + "type": "keyword" }, - "fields" : { - "properties" : { - "value" : { - "type" : "text" + "fields": { + "properties": { + "value": { + "type": "text" } } } @@ -529,7 +529,7 @@ } } }, - "pagerduty" : { + "pagerduty": { "type": "object", "dynamic": true, "properties": { @@ -544,55 +544,55 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, - "event" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "event": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "client" : { - "type" : "text" + "client": { + "type": "text" }, - "client_url" : { - "type" : "keyword" + "client_url": { + "type": "keyword" }, - "account" : { - "type" : "keyword" + "account": { + "type": "keyword" }, - "attach_payload" : { - "type" : "boolean" + "attach_payload": { + "type": "boolean" }, - "incident_key" : { - "type" : "keyword" + "incident_key": { + "type": "keyword" }, - "description" : { - "type" : "text" + "description": { + "type": "text" }, - "context" : { - "type" : "nested", + "context": { + "type": "nested", "include_in_parent": true, - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "href" : { - "type" : "keyword" + "href": { + "type": "keyword" }, - "src" : { - "type" : "keyword" + "src": { + "type": "keyword" }, - "alt" : { - "type" : "text" + "alt": { + "type": "text" } } } diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index 109ca95d75190..f5b89a179fd60 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -1,16 +1,16 @@ { "index_patterns": [ ".watcher-history-${xpack.watcher.template.version}*" ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.lifecycle.name": "watch-history-ilm-policy", - "index.hidden": true, - "index.format": 6 - }, - "mappings": { - "_doc": { + "priority": 2147483647, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.lifecycle.name": "watch-history-ilm-policy", + "index.hidden": true, + "index.format": 6 + }, + "mappings": { "_meta": { "watcher-history-version": "${xpack.watcher.template.version}" }, @@ -47,7 +47,7 @@ }, { "disabled_jira_custom_fields": { - "path_match": "result.actions.jira.fields.customfield_*", + "path_match": "result.actions.jira.fields.customfield_*", "mapping": { "type": "object", "enabled": false @@ -67,8 +67,8 @@ "type": "object", "dynamic": true, "properties": { - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, "triggered_time": { "type": "date" @@ -99,9 +99,9 @@ } } }, - "vars" : { - "type" : "object", - "enabled" : false + "vars": { + "type": "object", + "enabled": false }, "input": { "type": "object", @@ -116,8 +116,8 @@ }, "status": { "type": "object", - "enabled" : false, - "dynamic" : true + "enabled": false, + "dynamic": true }, "messages": { "type": "text" @@ -125,9 +125,9 @@ "user": { "type": "text" }, - "exception" : { - "type" : "object", - "enabled" : false + "exception": { + "type": "object", + "enabled": false }, "result": { "type": "object", @@ -143,15 +143,15 @@ "type": "object", "dynamic": true, "properties": { - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "payload" : { - "type" : "object", - "enabled" : false + "payload": { + "type": "object", + "enabled": false }, "search": { "type": "object", @@ -194,53 +194,53 @@ } } }, - "condition" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "condition": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "met" : { - "type" : "boolean" + "met": { + "type": "boolean" }, - "compare" : { - "type" : "object", - "enabled" : false + "compare": { + "type": "object", + "enabled": false }, - "array_compare" : { - "type" : "object", - "enabled" : false + "array_compare": { + "type": "object", + "enabled": false }, - "script" : { - "type" : "object", - "enabled" : false + "script": { + "type": "object", + "enabled": false } } }, - "transform" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "transform": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "search" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "request" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "indices" : { - "type" : "keyword" + "search": { + "type": "object", + "dynamic": true, + "properties": { + "request": { + "type": "object", + "dynamic": true, + "properties": { + "indices": { + "type": "keyword" }, - "types" : { - "type" : "keyword" + "types": { + "type": "keyword" } } } @@ -253,24 +253,24 @@ "include_in_parent": true, "dynamic": true, "properties": { - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" }, - "type" : { - "type" : "keyword" + "type": { + "type": "keyword" }, - "status" : { - "type" : "keyword" + "status": { + "type": "keyword" }, - "reason" : { - "type" : "keyword" + "reason": { + "type": "keyword" }, "number_of_actions_executed": { "type": "integer" }, - "foreach" : { + "foreach": { "type": "object", - "enabled" : false + "enabled": false }, "email": { "type": "object", @@ -341,7 +341,7 @@ } } }, - "jira" : { + "jira": { "type": "object", "dynamic": true, "properties": { @@ -351,13 +351,13 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, "fields": { "type": "object", @@ -369,30 +369,30 @@ "description": { "type": "text" }, - "labels" : { + "labels": { "type": "text" }, - "project" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "key" : { - "type" : "keyword" + "project": { + "type": "object", + "dynamic": true, + "properties": { + "key": { + "type": "keyword" }, - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" } } }, - "issuetype" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "name" : { + "issuetype": { + "type": "object", + "dynamic": true, + "properties": { + "name": { "type": "keyword" }, - "id" : { - "type" : "keyword" + "id": { + "type": "keyword" } } } @@ -401,21 +401,21 @@ "result": { "type": "object", "dynamic": true, - "properties" : { - "id" : { - "type" : "keyword" + "properties": { + "id": { + "type": "keyword" }, - "key" : { - "type" : "keyword" + "key": { + "type": "keyword" }, - "self" : { - "type" : "keyword" + "self": { + "type": "keyword" } } } } }, - "slack" : { + "slack": { "type": "object", "dynamic": true, "properties": { @@ -433,42 +433,42 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, - "to" : { + "to": { "type": "keyword" }, - "message" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "from" : { - "type" : "text" + "message": { + "type": "object", + "dynamic": true, + "properties": { + "from": { + "type": "text" }, - "icon" : { - "type" : "keyword" + "icon": { + "type": "keyword" }, - "text" : { - "type" : "text" + "text": { + "type": "text" }, - "attachments" : { - "type" : "nested", + "attachments": { + "type": "nested", "include_in_parent": true, - "dynamic" : true, - "properties" : { - "color" : { - "type" : "keyword" + "dynamic": true, + "properties": { + "color": { + "type": "keyword" }, - "fields" : { - "properties" : { - "value" : { - "type" : "text" + "fields": { + "properties": { + "value": { + "type": "text" } } } @@ -480,7 +480,7 @@ } } }, - "pagerduty" : { + "pagerduty": { "type": "object", "dynamic": true, "properties": { @@ -495,55 +495,55 @@ "reason": { "type": "text" }, - "request" : { - "type" : "object", - "enabled" : false + "request": { + "type": "object", + "enabled": false }, - "response" : { - "type" : "object", - "enabled" : false + "response": { + "type": "object", + "enabled": false }, - "event" : { - "type" : "object", - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "event": { + "type": "object", + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "client" : { - "type" : "text" + "client": { + "type": "text" }, - "client_url" : { - "type" : "keyword" + "client_url": { + "type": "keyword" }, - "account" : { - "type" : "keyword" + "account": { + "type": "keyword" }, - "attach_payload" : { - "type" : "boolean" + "attach_payload": { + "type": "boolean" }, - "incident_key" : { - "type" : "keyword" + "incident_key": { + "type": "keyword" }, - "description" : { - "type" : "text" + "description": { + "type": "text" }, - "context" : { - "type" : "nested", + "context": { + "type": "nested", "include_in_parent": true, - "dynamic" : true, - "properties" : { - "type" : { - "type" : "keyword" + "dynamic": true, + "properties": { + "type": { + "type": "keyword" }, - "href" : { - "type" : "keyword" + "href": { + "type": "keyword" }, - "src" : { - "type" : "keyword" + "src": { + "type": "keyword" }, - "alt" : { - "type" : "text" + "alt": { + "type": "text" } } } diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index 5181dfdc864f8..472ec45e44932 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -1,62 +1,62 @@ { "index_patterns": [ ".watches*" ], - "order": 2147483647, - "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, - "index.auto_expand_replicas": "0-1", - "index.format": 6, - "index.priority": 800 - }, - "mappings": { - "_doc": { - "dynamic" : "strict", + "priority": 2147483647, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.auto_expand_replicas": "0-1", + "index.format": 6, + "index.priority": 800 + }, + "mappings": { + "dynamic": "strict", "properties": { - "status": { - "type": "object", - "enabled" : false, - "dynamic" : true - }, - "trigger" : { - "type": "object", - "enabled" : false, - "dynamic" : true - }, - "input": { - "type": "object", - "enabled" : false, - "dynamic" : true - }, - "condition": { - "type": "object", - "enabled" : false, - "dynamic" : true - }, - "throttle_period": { - "type" : "keyword", - "index" : false, - "doc_values" : false - }, - "throttle_period_in_millis": { - "type" : "long", - "index" : false, - "doc_values" : false - }, - "transform": { - "type" : "object", - "enabled" : false, - "dynamic" : true - }, - "actions": { - "type" : "object", - "enabled" : false, - "dynamic" : true - }, - "metadata" : { - "type" : "object", - "dynamic": true + "status": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "trigger": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "input": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "condition": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "throttle_period": { + "type": "keyword", + "index": false, + "doc_values": false + }, + "throttle_period_in_millis": { + "type": "long", + "index": false, + "doc_values": false + }, + "transform": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "actions": { + "type": "object", + "enabled": false, + "dynamic": true + }, + "metadata": { + "type": "object", + "dynamic": true + } } - } } }, "version": ${xpack.watcher.template.version} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java index 7e9e3c900a871..07b42d3dbcf91 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java @@ -105,7 +105,7 @@ public void testDisabledDoesNotAddTemplates() { Settings settings = Settings.builder().put(SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false).build(); SnapshotLifecycleTemplateRegistry disabledRegistry = new SnapshotLifecycleTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry); - assertThat(disabledRegistry.getLegacyTemplateConfigs(), hasSize(0)); + assertThat(disabledRegistry.getComposableTemplateConfigs(), hasSize(0)); assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0)); } @@ -118,7 +118,7 @@ public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); calledTimes.set(0); @@ -232,7 +232,7 @@ public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); } public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { @@ -243,7 +243,7 @@ public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size()))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 323828e6e8a1c..271d4215b20ef 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -62,7 +62,7 @@ public ILMHistoryTemplateRegistry(Settings nodeSettings, ClusterService clusterS } @Override - protected List getLegacyTemplateConfigs() { + protected List getComposableTemplateConfigs() { if (this.ilmHistoryEnabled) { return Collections.singletonList(TEMPLATE_ILM_HISTORY); } else { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index 5c5237be631b5..c7d3ad744908f 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -64,7 +64,7 @@ public WatcherIndexTemplateRegistry(Settings nodeSettings, ClusterService cluste } @Override - protected List getLegacyTemplateConfigs() { + protected List getComposableTemplateConfigs() { return templatesToUse; } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index f975c87f4098a..a9fdbad3ed392 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; @@ -66,6 +67,7 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -115,8 +117,9 @@ public void testThatNonExistingTemplatesAreAddedImmediately() { ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); + verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry Map existingTemplates = new HashMap<>(); @@ -124,13 +127,13 @@ public void testThatNonExistingTemplatesAreAddedImmediately() { existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); - ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client.admin().indices(), times(4)).putTemplate(captor.capture(), anyObject()); - PutIndexTemplateRequest req = captor.getAllValues().stream() + argumentCaptor = ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); + verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + PutComposableIndexTemplateAction.Request req = argumentCaptor.getAllValues().stream() .filter(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)) .findFirst() .orElseThrow(() -> new AssertionError("expected the watch history template to be put")); - assertThat(req.settings().get("index.lifecycle.name"), equalTo("watch-history-ilm-policy")); + assertThat(req.indexTemplate().template().settings().get("index.lifecycle.name"), equalTo("watch-history-ilm-policy")); } public void testThatNonExistingTemplatesAreAddedEvenWithILMUsageDisabled() { @@ -142,8 +145,9 @@ public void testThatNonExistingTemplatesAreAddedEvenWithILMUsageDisabled() { clusterService, threadPool, client, xContentRegistry); ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes); registry.clusterChanged(event); - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); + verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry Map existingTemplates = new HashMap<>(); @@ -152,7 +156,7 @@ public void testThatNonExistingTemplatesAreAddedEvenWithILMUsageDisabled() { ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client.admin().indices(), times(5)).putTemplate(captor.capture(), anyObject()); + verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); captor.getAllValues().forEach(req -> assertNull(req.settings().get("index.lifecycle.name"))); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -264,9 +268,11 @@ public void testThatTemplatesAreAppliedOnNewerNodes() { ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); - verify(client.admin().indices(), times(1)).putTemplate(argumentCaptor.capture(), anyObject()); - assertThat(argumentCaptor.getValue().name(), is(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)); + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class); + verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject()); + assertTrue(argumentCaptor.getAllValues().stream() + .anyMatch(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME))); } public void testThatTemplatesAreNotAppliedOnSameVersionNodes() { From 1361e7e84820f17380ce4c13e2174d3d761c4a02 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 24 Aug 2020 20:48:05 +0200 Subject: [PATCH 02/11] fix watcher validate --- x-pack/docs/build.gradle | 1 + .../core/src/main/resources/ilm-history.json | 124 +++++++++--------- .../xpack/ilm/history/ILMHistoryStore.java | 2 + .../support/WatcherIndexTemplateRegistry.java | 8 +- 4 files changed, 69 insertions(+), 66 deletions(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index ff4ab76c0e0e1..291255260200d 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -60,6 +60,7 @@ testClusters.integTest { setting 'xpack.security.authc.realms.saml.saml1.sp.acs', 'https://kibana.org/api/security/saml/callback' setting 'xpack.security.authc.realms.saml.saml1.attributes.principal', 'uid' setting 'xpack.security.authc.realms.saml.saml1.attributes.name', 'urn:oid:2.5.4.3' + setting 'logger.org.elasticsearch.xpack.watcher', 'TRACE' user username: 'test_admin' } diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index b640e7eb2ed6a..05febc56ecd73 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -10,72 +10,72 @@ "index.lifecycle.name": "ilm-history-ilm-policy" }, "mappings": { - "dynamic": false, - "properties": { - "@timestamp": { - "type": "date", - "format": "epoch_millis" - }, - "policy": { - "type": "keyword" - }, - "index": { - "type": "keyword" - }, - "index_age": { - "type": "long" - }, - "success": { - "type": "boolean" - }, - "state": { - "type": "object", - "dynamic": true, - "properties": { - "phase": { - "type": "keyword" - }, - "action": { - "type": "keyword" - }, - "step": { - "type": "keyword" - }, - "failed_step": { - "type": "keyword" - }, - "is_auto-retryable_error": { - "type": "keyword" - }, - "creation_date": { - "type": "date", - "format": "epoch_millis" - }, - "phase_time": { - "type": "date", - "format": "epoch_millis" - }, - "action_time": { - "type": "date", - "format": "epoch_millis" - }, - "step_time": { - "type": "date", - "format": "epoch_millis" - }, - "phase_definition": { - "type": "text" - }, - "step_info": { - "type": "text" + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_millis" + }, + "policy": { + "type": "keyword" + }, + "index": { + "type": "keyword" + }, + "index_age": { + "type": "long" + }, + "success": { + "type": "boolean" + }, + "state": { + "type": "object", + "dynamic": true, + "properties": { + "phase": { + "type": "keyword" + }, + "action": { + "type": "keyword" + }, + "step": { + "type": "keyword" + }, + "failed_step": { + "type": "keyword" + }, + "is_auto-retryable_error": { + "type": "keyword" + }, + "creation_date": { + "type": "date", + "format": "epoch_millis" + }, + "phase_time": { + "type": "date", + "format": "epoch_millis" + }, + "action_time": { + "type": "date", + "format": "epoch_millis" + }, + "step_time": { + "type": "date", + "format": "epoch_millis" + }, + "phase_definition": { + "type": "text" + }, + "step_info": { + "type": "text" + } } + }, + "error_details": { + "type": "text" } - }, - "error_details": { - "type": "text" } } - } }, "_meta": { "description": "index template for ILM history indices", diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java index 4dc85f4362739..c11eb25f4c08c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java @@ -194,6 +194,8 @@ static void ensureHistoryIndex(Client client, ClusterState state, ActionListener byte[] templateBytes = TEMPLATE_ILM_HISTORY.loadBytes(); Map templateAsMap = XContentHelper.convertToMap(new BytesArray(templateBytes, 0, templateBytes.length), false, XContentType.JSON).v2(); + templateAsMap = (Map) templateAsMap.get("template"); + client.admin().indices().prepareCreate(initialHistoryIndexName) .setSettings((Map) templateAsMap.get("settings")) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index c7d3ad744908f..8cd33c084011c 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -85,10 +85,10 @@ protected String getOrigin() { } public static boolean validate(ClusterState state) { - return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) || - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && - state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); + return (state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) || + state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) && + state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) && + state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME); } } From 20c99fc1c337b8bb021c209a4ec140c32892031d Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 24 Aug 2020 20:56:46 +0200 Subject: [PATCH 03/11] fix watcher validate --- x-pack/docs/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 291255260200d..ff4ab76c0e0e1 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -60,7 +60,6 @@ testClusters.integTest { setting 'xpack.security.authc.realms.saml.saml1.sp.acs', 'https://kibana.org/api/security/saml/callback' setting 'xpack.security.authc.realms.saml.saml1.attributes.principal', 'uid' setting 'xpack.security.authc.realms.saml.saml1.attributes.name', 'urn:oid:2.5.4.3' - setting 'logger.org.elasticsearch.xpack.watcher', 'TRACE' user username: 'test_admin' } From 216805b9c0e384578a9028ea6a2584eef9476ada Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 25 Aug 2020 12:16:51 +0200 Subject: [PATCH 04/11] fix tests --- .../SnapshotLifecycleTemplateRegistry.java | 4 +-- ...napshotLifecycleTemplateRegistryTests.java | 36 +++++++++---------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index 6ec9a117574ab..fc8b496e5376e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -90,9 +90,9 @@ protected String getOrigin() { } public boolean validate(ClusterState state) { - boolean allTemplatesPresent = getLegacyTemplateConfigs().stream() + boolean allTemplatesPresent = getComposableTemplateConfigs().stream() .map(IndexTemplateConfig::getTemplateName) - .allMatch(name -> state.metadata().getTemplates().containsKey(name)); + .allMatch(name -> state.metadata().templatesV2().containsKey(name)); Optional> maybePolicies = Optional .ofNullable(state.metadata().custom(IndexLifecycleMetadata.TYPE)) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java index 07b42d3dbcf91..1d9b24129e2b1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java @@ -11,22 +11,20 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.TriFunction; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -148,7 +146,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { assertThat(putRequest.getPolicy().getName(), equalTo(SLM_POLICY_NAME)); assertNotNull(listener); return new PutLifecycleAction.Response(true); - } else if (action instanceof PutIndexTemplateAction) { + } else if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return new TestPutIndexTemplateResponse(true); } else { @@ -175,7 +173,7 @@ public void testPolicyAlreadyExists() { policyMap.put(policy.getName(), policy); client.setVerifier((action, request, listener) -> { - if (action instanceof PutIndexTemplateAction) { + if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return new TestPutIndexTemplateResponse(true); } else if (action instanceof PutLifecycleAction) { @@ -203,7 +201,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { LifecyclePolicy policy = policies.get(0); client.setVerifier((action, request, listener) -> { - if (action instanceof PutIndexTemplateAction) { + if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return new TestPutIndexTemplateResponse(true); } else if (action instanceof PutLifecycleAction) { @@ -255,7 +253,7 @@ public void testSameOrHigherVersionTemplateNotUpgraded() throws Exception { Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION), nodes); AtomicInteger calledTimes = new AtomicInteger(0); client.setVerifier((action, request, listener) -> { - if (action instanceof PutIndexTemplateAction) { + if (action instanceof PutComposableIndexTemplateAction) { fail("template should not have been re-installed"); return null; } else if (action instanceof PutLifecycleAction) { @@ -339,14 +337,14 @@ public VerifyingClient setVerifier(TriFunction, ActionRequest, Act private ActionResponse verifyTemplateInstalled( AtomicInteger calledTimes, ActionType action, ActionRequest request, ActionListener listener) { - if (action instanceof PutIndexTemplateAction) { + if (action instanceof PutComposableIndexTemplateAction) { calledTimes.incrementAndGet(); - assertThat(action, instanceOf(PutIndexTemplateAction.class)); - assertThat(request, instanceOf(PutIndexTemplateRequest.class)); - final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request; + assertThat(action, instanceOf(PutComposableIndexTemplateAction.class)); + assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class)); + final PutComposableIndexTemplateAction.Request putRequest = (PutComposableIndexTemplateAction.Request) request; assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME)); - assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); - assertThat(putRequest.version(), equalTo(INDEX_TEMPLATE_VERSION)); + assertThat(putRequest.indexTemplate().template().settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); + assertThat(putRequest.indexTemplate().version(), equalTo((long) INDEX_TEMPLATE_VERSION)); assertNotNull(listener); return new TestPutIndexTemplateResponse(true); } else if (action instanceof PutLifecycleAction) { @@ -376,11 +374,11 @@ private ClusterChangedEvent createClusterChangedEvent(Map exist private ClusterState createClusterState(Settings nodeSettings, Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { - ImmutableOpenMap.Builder indexTemplates = ImmutableOpenMap.builder(); + Map indexTemplates = new HashMap<>(); for (Map.Entry template : existingTemplates.entrySet()) { - final IndexTemplateMetadata mockTemplate = mock(IndexTemplateMetadata.class); - when(mockTemplate.version()).thenReturn(template.getValue()); - when(mockTemplate.getVersion()).thenReturn(template.getValue()); + final ComposableIndexTemplate mockTemplate = mock(ComposableIndexTemplate.class); + Long version = template.getValue() == null ? null : (long) template.getValue(); + when(mockTemplate.version()).thenReturn(version); indexTemplates.put(template.getKey(), mockTemplate); } @@ -391,7 +389,7 @@ private ClusterState createClusterState(Settings nodeSettings, Map Date: Tue, 25 Aug 2020 13:14:40 +0200 Subject: [PATCH 05/11] fix tests --- .../test/AbstractWatcherIntegrationTestCase.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 1c9d76210e9ed..131d1f0b08e4a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -467,12 +468,15 @@ protected void assertWatchWithMinimumActionsCount(final String watchName, final private void ensureWatcherTemplatesAdded() throws Exception { // Verify that the index templates exist: assertBusy(() -> { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(HISTORY_TEMPLATE_NAME).get(); - assertThat("[" + HISTORY_TEMPLATE_NAME + "] is missing", response.getIndexTemplates().size(), equalTo(1)); - response = client().admin().indices().prepareGetTemplates(TRIGGERED_TEMPLATE_NAME).get(); - assertThat("[" + TRIGGERED_TEMPLATE_NAME + "] is missing", response.getIndexTemplates().size(), equalTo(1)); - response = client().admin().indices().prepareGetTemplates(WATCHES_TEMPLATE_NAME).get(); - assertThat("[" + WATCHES_TEMPLATE_NAME + "] is missing", response.getIndexTemplates().size(), equalTo(1)); + GetComposableIndexTemplateAction.Response response = client().execute(GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(HISTORY_TEMPLATE_NAME)).get(); + assertThat("[" + HISTORY_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); + response = client().execute(GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(TRIGGERED_TEMPLATE_NAME)).get(); + assertThat("[" + TRIGGERED_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); + response = client().execute(GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(WATCHES_TEMPLATE_NAME)).get(); + assertThat("[" + WATCHES_TEMPLATE_NAME + "] is missing", response.indexTemplates().size(), equalTo(1)); }); } From 7ffdb8396fba8e8a59c3364ddf432e57a7c9802b Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 25 Aug 2020 16:24:52 +0200 Subject: [PATCH 06/11] unused imports --- .../xpack/watcher/test/AbstractWatcherIntegrationTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 131d1f0b08e4a..c9d19c3cac492 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; From 1fd73adec255f4bd212ac8b56e54535695cf9b7d Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Tue, 25 Aug 2020 22:44:16 +0200 Subject: [PATCH 07/11] fix test --- .../support/WatcherIndexTemplateRegistryTests.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index a9fdbad3ed392..2ac77c61fea00 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -356,11 +357,14 @@ private ClusterChangedEvent createClusterChangedEvent(Settings nodeSettings, private ClusterState createClusterState(Map existingTemplates) { Metadata.Builder metadataBuilder = Metadata.builder(); + HashMap templates = new HashMap<>(); for (Map.Entry template : existingTemplates.entrySet()) { - metadataBuilder.put(IndexTemplateMetadata.builder(template.getKey()) - .version(template.getValue()) - .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))); + ComposableIndexTemplate indexTemplate = mock(ComposableIndexTemplate.class); + when(indexTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + when(indexTemplate.indexPatterns()).thenReturn(Arrays.asList(generateRandomStringArray(10, 100, false, false))); + templates.put(template.getKey(), indexTemplate); } + metadataBuilder.indexTemplates(templates); return ClusterState.builder(new ClusterName("foo")).metadata(metadataBuilder.build()).build(); } From b9e18d1dbd95462189a6d120231017e1becac7a9 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 27 Aug 2020 12:47:42 +0200 Subject: [PATCH 08/11] added _meta, increased version --- .../watcher/support/WatcherIndexTemplateRegistryField.java | 3 ++- x-pack/plugin/core/src/main/resources/triggered-watches.json | 4 ++++ .../plugin/core/src/main/resources/watch-history-no-ilm.json | 4 ++++ x-pack/plugin/core/src/main/resources/watch-history.json | 4 ++++ x-pack/plugin/core/src/main/resources/watches.json | 4 ++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 2c3442956d113..82994613a8ade 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -16,8 +16,9 @@ public final class WatcherIndexTemplateRegistryField { // version 9: add a user field defining which user executed the watch // version 10: add support for foreach path in actions // version 11: watch history indices are hidden + // version 12: templates changed to composable templates // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 11; + public static final int INDEX_TEMPLATE_VERSION = 12; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index f0a8a4f418292..1cb43c51c6ef5 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -37,5 +37,9 @@ } } }, + "_meta": { + "description": "index template for triggered watches indices", + "managed": true + }, "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 185a47b96971b..2f4433536d4ae 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -613,5 +613,9 @@ } } }, + "_meta": { + "description": "index template for watcher history indices", + "managed": true + }, "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index f5b89a179fd60..c5b380165fd78 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -564,5 +564,9 @@ } } }, + "_meta": { + "description": "index template for watcher history indices", + "managed": true + }, "version": ${xpack.watcher.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index 472ec45e44932..ab9ba4b1d9ec6 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -59,5 +59,9 @@ } } }, + "_meta": { + "description": "index template for watches indices", + "managed": true + }, "version": ${xpack.watcher.template.version} } From 40e1a5459bb7b7577ee4470c973173ffe56f5bad Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 27 Aug 2020 12:48:33 +0200 Subject: [PATCH 09/11] increased version --- .../core/slm/history/SnapshotLifecycleTemplateRegistry.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index fc8b496e5376e..0628dcc4b8f93 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -36,7 +36,8 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial // version 2: converted to hidden index - public static final int INDEX_TEMPLATE_VERSION = 2; + // version 3: templates moved to composable templates + public static final int INDEX_TEMPLATE_VERSION = 3; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; From 7f852e43f3be33afd29e5e4477d9b3870c7e6374 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 27 Aug 2020 17:11:17 +0200 Subject: [PATCH 10/11] increased version --- .../xpack/ilm/history/ILMHistoryTemplateRegistry.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 271d4215b20ef..f14be385d89f3 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -28,7 +28,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial // version 2: convert to hidden index - public static final int INDEX_TEMPLATE_VERSION = 2; + // version 2: templates moved to composable templates + public static final int INDEX_TEMPLATE_VERSION = 3; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; From dacba2e5b1fbfb7bb3441bb614f0dc2ebdbdce11 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 27 Aug 2020 17:14:34 +0200 Subject: [PATCH 11/11] Update x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java Co-authored-by: Lee Hinman --- .../xpack/ilm/history/ILMHistoryTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index f14be385d89f3..08494b7fb70e8 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -28,7 +28,7 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial // version 2: convert to hidden index - // version 2: templates moved to composable templates + // version 3: templates moved to composable templates public static final int INDEX_TEMPLATE_VERSION = 3; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version";