diff --git a/docs/changelog/95666.yaml b/docs/changelog/95666.yaml new file mode 100644 index 0000000000000..5f8d5c62fc05e --- /dev/null +++ b/docs/changelog/95666.yaml @@ -0,0 +1,5 @@ +pr: 95666 +summary: Bootstrap profiling indices at startup +area: Indices APIs +type: enhancement +issues: [] diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 4fce5e99f4495..0fca813143911 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -232,7 +232,7 @@ snapshot as well as each feature's indices. To restore a specific feature state from the snapshot, specify the `feature_name` from the response in the restore snapshot API's -<> parameter. +<> parameter. NOTE: When you restore a feature state, {es} closes and overwrites the feature's existing indices. @@ -247,7 +247,7 @@ realm>> to ensure you'll still be able to access your cluster. POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore { "feature_states": [ "geoip" ], - "include_global_state": false, <1> + "include_global_state": false, <1> "indices": "-*" <2> } ---- @@ -256,7 +256,7 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore // TEST[s/_restore/_restore?wait_for_completion=true/] // TEST[s/"feature_states": \[ "geoip" \],//] -<1> Exclude the cluster state from the restore operation. +<1> Exclude the cluster state from the restore operation. <2> Exclude the other indices and data streams in the snapshot from the restore operation. [discrete] @@ -365,6 +365,27 @@ POST _watcher/_start . {blank} + -- +* Universal Profiling ++ +Check if Universal Profiling index template management is enabled: ++ +[source,console] +---- +GET /_cluster/settings?filter_path=**.xpack.profiling.templates.enabled&include_defaults=true +---- ++ +If the value is `true`, disable Universal Profiling index template management: ++ +[source,console] +---- +PUT _cluster/settings +{ + "persistent": { + "xpack.profiling.templates.enabled": false + } +} +---- + [[restore-create-file-realm-user]] If you use {es} security features, log in to a node host, navigate to the {es} installation directory, and add a user with the `superuser` role to the file @@ -480,6 +501,21 @@ POST _watcher/_start ---- -- +* Universal Profiling ++ +If the value was `true` initially, enable Universal Profiling index template management again, otherwise skip this step: ++ +[source,console] +---- +PUT _cluster/settings +{ + "persistent": { + "xpack.profiling.templates.enabled": true + } +} +---- +//TEST[s/true/false/] + . If wanted, reset the `action.destructive_requires_name` cluster setting. + [source,console] diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 40522422d6ba2..57c2ab1c64170 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -621,6 +621,7 @@ protected Set preserveILMPolicyIds() { "ml-size-based-ilm-policy", "logs", "metrics", + "profiling", "synthetics", "7-days-default", "30-days-default", @@ -1814,11 +1815,15 @@ protected static boolean isXPackTemplate(String name) { if (name.startsWith("behavioral_analytics-")) { return true; } + if (name.startsWith("profiling-")) { + return true; + } switch (name) { case ".watches": case "security_audit_log": case ".slm-history": case ".async-search": + case ".profiling-ilm-lock": // TODO: Remove after switch to K/V indices case "saml-service-provider": case "logs": case "logs-settings": diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index 37f5b32da1646..4cf8c72c087ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -186,6 +186,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion( public static final String TRANSFORM_ORIGIN = "transform"; public static final String ASYNC_SEARCH_ORIGIN = "async_search"; public static final String IDP_ORIGIN = "idp"; + public static final String PROFILING_ORIGIN = "profiling"; public static final String STACK_ORIGIN = "stack"; public static final String SEARCHABLE_SNAPSHOTS_ORIGIN = "searchable_snapshots"; public static final String LOGSTASH_MANAGEMENT_ORIGIN = "logstash_management"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 8e543f4466a8b..3f785eb6f2a04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -93,6 +93,13 @@ public Iterator> settings() { Setting.Property.NodeScope ); + /** Setting for enabling or disabling universal profiling. Defaults to true. */ + public static final Setting PROFILING_ENABLED = Setting.boolSetting( + "xpack.profiling.enabled", + true, + Setting.Property.NodeScope + ); + /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( "xpack.ent_search.enabled", @@ -318,6 +325,7 @@ public static List> getAllSettings() { settings.add(SECURITY_ENABLED); settings.add(GRAPH_ENABLED); settings.add(MACHINE_LEARNING_ENABLED); + settings.add(PROFILING_ENABLED); settings.add(ENTERPRISE_SEARCH_ENABLED); settings.add(AUDIT_ENABLED); settings.add(WATCHER_ENABLED); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java index b7d69bc160d2a..99c8df4d75ea3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/LifecyclePolicyConfig.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils; import org.elasticsearch.xpack.core.ilm.LifecycleType; import org.elasticsearch.xpack.core.ilm.RolloverAction; +import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; @@ -36,6 +37,7 @@ public class LifecyclePolicyConfig { (p) -> TimeseriesLifecycleType.INSTANCE ), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse), + new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(SetPriorityAction.NAME), SetPriorityAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ForceMergeAction.NAME), ForceMergeAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(ShrinkAction.NAME), ShrinkAction::parse), new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse) diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json similarity index 65% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/events.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json index 855c754191db5..e952e24bd5358 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-events.json @@ -1,24 +1,26 @@ { - "settings": { - "index": { - "number_of_shards": "4", - "max_result_window": 150000, - "refresh_interval": "10s", - "sort": { - "field": [ - "service.name", - "@timestamp", - "orchestrator.resource.name", - "container.name", - "process.thread.name", - "host.id" - ] - } + "template": { + "settings": { + "index": { + "number_of_shards": 4, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "max_result_window": 150000, + "refresh_interval": "10s", + "sort": { + "field": [ + "service.name", + "@timestamp", + "orchestrator.resource.name", + "container.name", + "process.thread.name", + "host.id" + ] + } + }, + "codec": "best_compression" }, - "codec": "best_compression" - }, - "mappings": { - "_doc": { + "mappings": { "_source": { "enabled": false }, @@ -60,9 +62,6 @@ "host.ip": { "type": "ip" }, - "host.ipstring": { - "type": "keyword" - }, "host.name": { "type": "keyword" }, @@ -74,5 +73,6 @@ } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json similarity index 55% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json index f5db60318bcf2..2fc9774200f47 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json @@ -1,11 +1,13 @@ { - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_doc": { + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s" + } + }, + "mappings": { "_source": { "mode": "synthetic" }, @@ -25,8 +27,14 @@ "@timestamp": { "type": "date", "format": "epoch_second" + }, + "Symbolization.lastprocessed": { + "type": "date", + "format": "epoch_second", + "index": false } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json new file mode 100644 index 0000000000000..42fbad496c0e9 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json @@ -0,0 +1,12 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "profiling" + } + } + } + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json new file mode 100644 index 0000000000000..2a4895b2fea0f --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json @@ -0,0 +1,44 @@ +{ + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "sort": { + "field": [ + "project.id", + "@timestamp", + "host.id" + ] + } + }, + "codec": "best_compression" + }, + "mappings": { + "_source": { + "enabled": false + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "project.id": { + "type": "keyword" + }, + "host.id": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "Elasticsearch.cluster.id": { + "type": "keyword" + } + } + } + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json similarity index 62% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json index ced7b081a3d36..a238c7805b549 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json @@ -1,12 +1,14 @@ { - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_doc": { + "template": { + "settings": { + "index": { + "number_of_shards": 16, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s" + } + }, + "mappings": { "_source": { "enabled": true }, @@ -34,8 +36,15 @@ "index": false, "doc_values": false, "store": false + }, + "Stackframe.function.offset": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json similarity index 52% rename from x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json rename to x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json index 0b72dccdfe5c5..7f08399e12fe9 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json @@ -1,17 +1,19 @@ { - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s", - "sort": { - "field": [ - "Stacktrace.frame.ids" - ] + "template": { + "settings": { + "index": { + "number_of_shards": 16, + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "sort": { + "field": [ + "Stacktrace.frame.ids" + ] + } } - } - }, - "mappings": { - "_doc": { + }, + "mappings": { "_source": { "mode": "synthetic" }, @@ -27,12 +29,9 @@ "Stacktrace.frame.types": { "type": "keyword", "index": false - }, - "@timestamp": { - "type": "date", - "format": "epoch_second" } } } - } + }, + "version": ${xpack.profiling.template.version} } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json new file mode 100644 index 0000000000000..9d01e97e430d2 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json @@ -0,0 +1,38 @@ +{ + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "7d", + "min_docs": 1 + }, + "set_priority": { + "priority": 100 + } + } + }, + "warm": { + "min_age": "30d", + "actions": { + "set_priority": { + "priority": 50 + }, + "shrink": { + "number_of_shards": 2 + } + } + }, + "delete": { + "min_age": "60d", + "actions": { + "delete": {} + } + } + }, + "_meta": { + "description": "default ILM policy for Elastic Universal Profiling", + "managed": true + } +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json new file mode 100644 index 0000000000000..84f6c013f6356 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json @@ -0,0 +1,30 @@ +{ + "index_patterns": [ + ".profiling-ilm-lock*" + ], + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "hidden": true + } + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "phase": { + "type": "keyword" + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-ilm-lock" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json new file mode 100644 index 0000000000000..2ade70d6a0a81 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-events.json @@ -0,0 +1,17 @@ +{ + "index_patterns": [ + "profiling-events*" + ], + "data_stream": { + "hidden": false + }, + "composed_of": [ + "profiling-events", + "profiling-ilm" + ], + "priority": 100, + "_meta": { + "description": "Index template for profiling-events" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json new file mode 100644 index 0000000000000..504d870e3f754 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json @@ -0,0 +1,13 @@ +{ + "index_patterns": [ + "profiling-executables*" + ], + "composed_of": [ + "profiling-executables" + ], + "priority": 100, + "_meta": { + "description": "Index template for profiling-executables" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json new file mode 100644 index 0000000000000..080bd7d5bcef4 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json @@ -0,0 +1,15 @@ +{ + "index_patterns": [ + "profiling-metrics*" + ], + "data_stream": {}, + "composed_of": [ + "profiling-metrics", + "profiling-ilm" + ], + "priority": 100, + "_meta": { + "description": "Template for profiling-metrics" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json new file mode 100644 index 0000000000000..dc2508e2853c9 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json @@ -0,0 +1,68 @@ +{ + "index_patterns": [ + ".profiling-returnpads-private*" + ], + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "hidden": true + } + }, + "mappings": { + /* + Enable source as the main purpose of this index is to store relatively few BLOBs. + Disabling source is another option, but adds the 'recovery_source' extra storage costs. + */ + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.created": { + "type": "date", + "doc_values": false, + "index": true, + "store": false + }, + "Symbfile.file.id": { + /* 'binary' type fields don't allow using 'index: true'. */ + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + "Symbfile.part": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + /* This is the number of parts for the file.id. It only exists in the last part. */ + "Symbfile.parts": { + "type": "short", + "index": false, + "doc_values": false, + "store": false + }, + "Symbfile.data": { + "type": "binary", + "doc_values": false, + "store": false + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-returnpads-private" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json new file mode 100644 index 0000000000000..602cddb665391 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json @@ -0,0 +1,48 @@ +{ + "index_patterns": [ + ".profiling-sq-executables*" + ], + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "hidden": true + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Executable.file.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-sq-executables" + }, + "version": ${xpack.profiling.template.version} +} + diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json new file mode 100644 index 0000000000000..2749d2746408d --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json @@ -0,0 +1,47 @@ +{ + "index_patterns": [ + ".profiling-sq-leafframes*" + ], + "template": { + "settings": { + "index": { + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "hidden": true + } + }, + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true + }, + "Stacktrace.frame.id": { + "type": "keyword", + "index": false + }, + "Time.created": { + "type": "date", + "index": true + }, + "Symbolization.time.next": { + "type": "date", + "index": true + }, + "Symbolization.retries": { + "type": "short", + "index": true + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-sq-leafframes" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json new file mode 100644 index 0000000000000..6eba29855b075 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json @@ -0,0 +1,13 @@ +{ + "index_patterns": [ + "profiling-stackframes*" + ], + "composed_of": [ + "profiling-stackframes" + ], + "priority": 100, + "_meta": { + "description": "Index template for profiling-stackframes" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json new file mode 100644 index 0000000000000..2e169e19b72d5 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json @@ -0,0 +1,13 @@ +{ + "index_patterns": [ + "profiling-stacktraces*" + ], + "composed_of": [ + "profiling-stacktraces" + ], + "priority": 100, + "_meta": { + "description": "Index template for profiling-stacktraces" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json new file mode 100644 index 0000000000000..e266cb4e59896 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json @@ -0,0 +1,130 @@ +{ + "index_patterns": [ + ".profiling-symbols*" + ], + "template": { + "settings": { + "index": { + "number_of_shards": "16", + "number_of_replicas": 0, + "auto_expand_replicas": "0-1", + "refresh_interval": "10s", + "hidden": true + } + }, + "mappings": { + /* + Enable source for now because ip_range is not supported by synthetic source. + And we currently want to use the reindex API for experiments with storage costs. + */ + "_source": { + "enabled": true + }, + "properties": { + "ecs.version": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + /* name of the function */ + "Symbol.function.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + /* file path */ + "Symbol.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + /* (for inlined functions) file path where inline function was called */ + "Symbol.call.file.name": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + /* (for inlined functions) line where inline function was called */ + "Symbol.call.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + /* function start line (only available from DWARF). Currently unused. */ + "Symbol.function.line": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + /* inline depth */ + "Symbol.depth": { + "type": "integer", + "index": false, + "doc_values": false, + "store": false + }, + /* + pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end. + To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number. + offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case. + Ranges are stored as points, which cannot be retrieve when disabling _source. + See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html . + + Linetable: base for offsets (64bit PC range base) + */ + "Symbol.linetable.base": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + /* Linetable: length of range (PC range is [base, base+length)) */ + "Symbol.linetable.length": { + "type": "unsigned_long", + "index": false, + "doc_values": false, + "store": false + }, + /* Linetable: concatenated offsets (each value is ULEB128encoded) */ + "Symbol.linetable.offsets": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + /* Linetable: concatenated lines (each value is ULEB128 encoded) */ + "Symbol.linetable.lines": { + "type": "keyword", + "index": false, + "doc_values": false, + "store": false + }, + /* fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization */ + "Symbol.file.id": { + "type": "keyword", + "index": true, + "doc_values": false, + "store": false + }, + /* PC ranges [begin, end) */ + "Symbol.exec.pcrange": { + "type": "ip_range", + "index": true, + "doc_values": false, + "store": false + } + } + } + }, + "priority": 100, + "_meta": { + "description": "Index template for .profiling-symbols" + }, + "version": ${xpack.profiling.template.version} +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java index d545bef0e1e7a..261b9d10a47dd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java @@ -41,6 +41,7 @@ protected Settings nodeSettings() { .put(XPackSettings.WATCHER_ENABLED.getKey(), false) .put(XPackSettings.GRAPH_ENABLED.getKey(), false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java index f717dfe50639d..4bee07f3f5701 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java @@ -48,6 +48,7 @@ protected Settings localExporterSettings() { .put("xpack.monitoring.exporters." + exporterName + ".enabled", false) .put("xpack.monitoring.exporters." + exporterName + ".cluster_alerts.management.enabled", false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) + .put(XPackSettings.PROFILING_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/profiler/build.gradle b/x-pack/plugin/profiler/build.gradle index 80624ec9a1815..dc680693aca29 100644 --- a/x-pack/plugin/profiler/build.gradle +++ b/x-pack/plugin/profiler/build.gradle @@ -12,4 +12,13 @@ esplugin { name 'x-pack-profiling' description 'The profiler plugin adds support for retrieving data from Universal Profiler.' classname 'org.elasticsearch.xpack.profiler.ProfilingPlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(path: xpackModule('mapper-unsigned-long')) + testImplementation project(path: xpackModule('ilm')) + testImplementation project(':modules:data-streams') } diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java index e79a79d4bbc28..14b624d7bc1ec 100644 --- a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java +++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java @@ -7,17 +7,25 @@ package org.elasticsearch.xpack.profiler; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.ilm.IndexLifecycle; +import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; +import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; @@ -26,7 +34,14 @@ public abstract class ProfilingTestCase extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(ProfilingPlugin.class, getTestTransportPlugin()); + return List.of( + LocalStateCompositeXPackPlugin.class, + DataStreamsPlugin.class, + ProfilingPlugin.class, + IndexLifecycle.class, + UnsignedLongMapperPlugin.class, + getTestTransportPlugin() + ); } @Override @@ -35,6 +50,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) + .put(XPackSettings.PROFILING_ENABLED.getKey(), true) + .put(ProfilingPlugin.PROFILING_TEMPLATES_ENABLED.getKey(), false) + // .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") + // Disable ILM history index so that the tests don't have to clean it up + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.getKey(), false) .build(); } @@ -48,16 +68,8 @@ protected boolean ignoreExternalCluster() { return true; } - private byte[] read(String resource) throws IOException { - return GetProfilingAction.class.getClassLoader().getResourceAsStream(resource).readAllBytes(); - } - - private void createIndex(String name, String bodyFileName) throws Exception { - client().admin().indices().prepareCreate(name).setSource(read(bodyFileName), XContentType.JSON).execute().get(); - } - private void indexDoc(String index, String id, Map source) { - IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).get(); + IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).setCreate(true).get(); assertEquals(RestStatus.CREATED, indexResponse.status()); } @@ -71,16 +83,29 @@ private void indexDoc(String index, String id, Map source) { */ protected abstract boolean useOnlyAllEvents(); + protected void waitForIndices() throws Exception { + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().get().getState(); + assertTrue( + "Timed out waiting for the indices to be created", + state.metadata().indices().keySet().containsAll(ProfilingIndexManager.INDICES_AND_ALIASES.keySet()) + ); + }); + } + + protected void updateProfilingTemplatesEnabled(boolean newValue) { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + request.persistentSettings(Settings.builder().put(ProfilingPlugin.PROFILING_TEMPLATES_ENABLED.getKey(), newValue).build()); + ClusterUpdateSettingsResponse response = client().admin().cluster().updateSettings(request).actionGet(); + assertTrue("Update of profiling templates enabled setting is not acknowledged", response.isAcknowledged()); + } + @Before public void setupData() throws Exception { + // only enable index management while setting up indices to avoid interfering with the rest of the test infrastructure + updateProfilingTemplatesEnabled(true); Collection eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames(); - - for (String idx : eventsIndices) { - createIndex(idx, "events.json"); - } - createIndex("profiling-stackframes", "stackframes.json"); - createIndex("profiling-stacktraces", "stacktraces.json"); - createIndex("profiling-executables", "executables.json"); + waitForIndices(); ensureGreen(); // ensure that we have this in every index, so we find an event @@ -106,4 +131,9 @@ public void setupData() throws Exception { refresh(); } + + @After + public void disable() { + updateProfilingTemplatesEnabled(false); + } } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java new file mode 100644 index 0000000000000..ea5748be4c334 --- /dev/null +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexManager.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ClientHelper; + +import java.io.Closeable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +/** + * Creates all indices that are required for using Elastic Universal Profiling. + */ +public class ProfilingIndexManager implements ClusterStateListener, Closeable { + private static final Logger logger = LogManager.getLogger(ProfilingIndexManager.class); + // For testing + public static final Map INDICES_AND_ALIASES; + + static { + String versionSuffix = "-v" + ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION; + + Map indicesAndAliases = new HashMap<>(); + // TODO: Define behavior on upgrade (delete, reindex, ...), to be done after 8.9.0 + // TODO: This index will be gone with the 8.9 release. Don't bother to implement versioning support. + indicesAndAliases.put(".profiling-ilm-lock", null); + indicesAndAliases.put(".profiling-returnpads-private" + versionSuffix, "profiling-returnpads-private"); + indicesAndAliases.put(".profiling-sq-executables" + versionSuffix, "profiling-sq-executables"); + indicesAndAliases.put(".profiling-sq-leafframes" + versionSuffix, "profiling-sq-leafframes"); + indicesAndAliases.put(".profiling-symbols" + versionSuffix, "profiling-symbols"); + indicesAndAliases.put(".profiling-symbols-private" + versionSuffix, "profiling-symbols-private"); + // TODO: Update these to the new K/V strategy after all readers have been adjusted + String[] kvIndices = new String[] { "profiling-executables", "profiling-stackframes", "profiling-stacktraces" }; + for (String idx : kvIndices) { + indicesAndAliases.put(idx + versionSuffix + "-000001", idx); + indicesAndAliases.put(idx + versionSuffix + "-000002", idx + "-next"); + } + INDICES_AND_ALIASES = Collections.unmodifiableMap(indicesAndAliases); + } + + private final ThreadPool threadPool; + private final Client client; + private final ClusterService clusterService; + private final ConcurrentMap creationInProgressPerIndex = new ConcurrentHashMap<>(); + private volatile boolean templatesEnabled; + + public ProfilingIndexManager(ThreadPool threadPool, Client client, ClusterService clusterService) { + this.threadPool = threadPool; + this.client = client; + this.clusterService = clusterService; + } + + public void initialize() { + clusterService.addListener(this); + } + + @Override + public void close() { + clusterService.removeListener(this); + } + + public void setTemplatesEnabled(boolean templatesEnabled) { + this.templatesEnabled = templatesEnabled; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (templatesEnabled == false) { + return; + } + // wait for the cluster state to be recovered + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + return; + } + + // If this node is not a master node, exit. + if (event.state().nodes().isLocalNodeElectedMaster() == false) { + return; + } + + if (event.state().nodes().getMaxNodeVersion().after(event.state().nodes().getSmallestNonClientNodeVersion())) { + logger.debug("Skipping up-to-date check as cluster has mixed versions"); + return; + } + + // ensure that index templates are present + if (isAllTemplatesCreated(event) == false) { + logger.trace("Skipping index creation; not all templates are present yet"); + return; + } + + addIndicesIfMissing(event.state()); + } + + protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { + return ProfilingIndexTemplateRegistry.areAllTemplatesCreated(event.state()); + } + + private void addIndicesIfMissing(ClusterState state) { + Map indicesMetadata = state.metadata().indices(); + for (Map.Entry idxAlias : INDICES_AND_ALIASES.entrySet()) { + String index = idxAlias.getKey(); + String alias = idxAlias.getValue(); + final AtomicBoolean creationInProgress = creationInProgressPerIndex.computeIfAbsent(index, key -> new AtomicBoolean(false)); + if (creationInProgress.compareAndSet(false, true)) { + final boolean indexNeedsToBeCreated = indicesMetadata == null || indicesMetadata.get(index) == null; + if (indexNeedsToBeCreated) { + logger.debug("adding index [{}], because it doesn't exist", index); + putIndex(index, alias, creationInProgress); + } else { + logger.trace("not adding index [{}], because it already exists", index); + creationInProgress.set(false); + } + } + } + } + + private void onPutIndexFailure(String index, Exception ex) { + logger.error(() -> format("error adding index [%s] for [%s]", index, ClientHelper.PROFILING_ORIGIN), ex); + } + + private void putIndex(final String index, final String alias, final AtomicBoolean creationCheck) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + CreateIndexRequest request = new CreateIndexRequest(index); + if (alias != null) { + try { + Map sourceAsMap = Map.of("aliases", Map.of(alias, Map.of("is_write_index", true))); + request.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } catch (Exception ex) { + creationCheck.set(false); + onPutIndexFailure(index, ex); + return; + } + } + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + ClientHelper.PROFILING_ORIGIN, + request, + new ActionListener() { + @Override + public void onResponse(CreateIndexResponse response) { + creationCheck.set(false); + if (response.isAcknowledged() == false) { + logger.error( + "error adding index [{}] for [{}], request was not acknowledged", + index, + ClientHelper.PROFILING_ORIGIN + ); + } else if (response.isShardsAcknowledged() == false) { + logger.warn("adding index [{}] for [{}], shards were not acknowledged", index, ClientHelper.PROFILING_ORIGIN); + } + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onPutIndexFailure(index, e); + } + }, + (req, listener) -> client.admin().indices().create(req, listener) + ); + }); + } +} diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java new file mode 100644 index 0000000000000..2a4e616001a0b --- /dev/null +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistry.java @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.template.IndexTemplateConfig; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Creates all index-templates and ILM policies that are required for using Elastic Universal Profiling. + */ +public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { + // history (please add a comment why you increased the version here) + // version 1: initial + public static final int INDEX_TEMPLATE_VERSION = 1; + + public static final String PROFILING_TEMPLATE_VERSION_VARIABLE = "xpack.profiling.template.version"; + + private volatile boolean templatesEnabled; + + public ProfilingIndexTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + } + + public void setTemplatesEnabled(boolean templatesEnabled) { + this.templatesEnabled = templatesEnabled; + } + + public void close() { + clusterService.removeListener(this); + } + + @Override + protected String getOrigin() { + return ClientHelper.PROFILING_ORIGIN; + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + private static final List LIFECYCLE_POLICIES = List.of( + new LifecyclePolicyConfig("profiling", "/org/elasticsearch/xpack/profiler/ilm-policy/profiling.json").load( + LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY + ) + ); + + @Override + protected List getPolicyConfigs() { + return templatesEnabled ? LIFECYCLE_POLICIES : Collections.emptyList(); + } + + private static final Map COMPONENT_TEMPLATE_CONFIGS; + + static { + final Map componentTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + "profiling-events", + "/org/elasticsearch/xpack/profiler/component-template/profiling-events.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-executables", + "/org/elasticsearch/xpack/profiler/component-template/profiling-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-ilm", + "/org/elasticsearch/xpack/profiler/component-template/profiling-ilm.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-metrics", + "/org/elasticsearch/xpack/profiler/component-template/profiling-metrics.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stackframes", + "/org/elasticsearch/xpack/profiler/component-template/profiling-stackframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stacktraces", + "/org/elasticsearch/xpack/profiler/component-template/profiling-stacktraces.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ) + )) { + try { + componentTemplates.put( + config.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) + ); + } catch (IOException e) { + throw new AssertionError(e); + } + } + COMPONENT_TEMPLATE_CONFIGS = Collections.unmodifiableMap(componentTemplates); + } + + @Override + protected Map getComponentTemplateConfigs() { + return templatesEnabled ? COMPONENT_TEMPLATE_CONFIGS : Collections.emptyMap(); + } + + private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( + new IndexTemplateConfig( + "profiling-events", + "/org/elasticsearch/xpack/profiler/index-template/profiling-events.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-metrics", + "/org/elasticsearch/xpack/profiler/index-template/profiling-metrics.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-executables", + "/org/elasticsearch/xpack/profiler/index-template/profiling-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stackframes", + "/org/elasticsearch/xpack/profiler/index-template/profiling-stackframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-stacktraces", + "/org/elasticsearch/xpack/profiler/index-template/profiling-stacktraces.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + // templates for regular indices + new IndexTemplateConfig( + ".profiling-ilm-lock", + "/org/elasticsearch/xpack/profiler/index-template/.profiling-ilm-lock.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-returnpads-private", + "/org/elasticsearch/xpack/profiler/index-template/profiling-returnpads-private.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-sq-executables", + "/org/elasticsearch/xpack/profiler/index-template/profiling-sq-executables.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-sq-leafframes", + "/org/elasticsearch/xpack/profiler/index-template/profiling-sq-leafframes.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ), + new IndexTemplateConfig( + "profiling-symbols", + "/org/elasticsearch/xpack/profiler/index-template/profiling-symbols.json", + INDEX_TEMPLATE_VERSION, + PROFILING_TEMPLATE_VERSION_VARIABLE + ) + ); + + @Override + protected Map getComposableTemplateConfigs() { + return templatesEnabled ? COMPOSABLE_INDEX_TEMPLATE_CONFIGS : Collections.emptyMap(); + } + + public static boolean areAllTemplatesCreated(ClusterState state) { + for (String componentTemplate : COMPONENT_TEMPLATE_CONFIGS.keySet()) { + if (state.metadata().componentTemplates().containsKey(componentTemplate) == false) { + return false; + } + } + for (String composableTemplate : COMPOSABLE_INDEX_TEMPLATE_CONFIGS.keySet()) { + if (state.metadata().templatesV2().containsKey(composableTemplate) == false) { + return false; + } + } + return true; + } +} diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java index 7c26cf252de32..471a843bc2b95 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; @@ -37,6 +38,7 @@ import org.elasticsearch.tracing.Tracer; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.core.XPackSettings; import java.util.Collection; import java.util.Collections; @@ -47,17 +49,23 @@ public class ProfilingPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class); - public static final Setting PROFILING_ENABLED = Setting.boolSetting( - "xpack.profiling.enabled", - true, - Setting.Property.NodeScope + public static final Setting PROFILING_TEMPLATES_ENABLED = Setting.boolSetting( + "xpack.profiling.templates.enabled", + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic ); public static final String PROFILING_THREAD_POOL_NAME = "profiling"; - + private final Settings settings; private final boolean enabled; + private final SetOnce registry = new SetOnce<>(); + + private final SetOnce indexManager = new SetOnce<>(); + public ProfilingPlugin(Settings settings) { - this.enabled = PROFILING_ENABLED.get(settings); + this.settings = settings; + this.enabled = XPackSettings.PROFILING_ENABLED.get(settings); } @Override @@ -77,21 +85,26 @@ public Collection createComponents( AllocationService allocationService ) { logger.info("Profiling is {}", enabled ? "enabled" : "disabled"); - return super.createComponents( - client, - clusterService, - threadPool, - resourceWatcherService, - scriptService, - xContentRegistry, - environment, - nodeEnvironment, - namedWriteableRegistry, - indexNameExpressionResolver, - repositoriesServiceSupplier, - tracer, - allocationService - ); + registry.set(new ProfilingIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry)); + indexManager.set(new ProfilingIndexManager(threadPool, client, clusterService)); + // set initial value + updateTemplatesEnabled(PROFILING_TEMPLATES_ENABLED.get(settings)); + clusterService.getClusterSettings().addSettingsUpdateConsumer(PROFILING_TEMPLATES_ENABLED, this::updateTemplatesEnabled); + if (enabled) { + registry.get().initialize(); + indexManager.get().initialize(); + return List.of(registry.get(), indexManager.get()); + } else { + return Collections.emptyList(); + } + } + + public void updateTemplatesEnabled(boolean newValue) { + if (newValue == false) { + logger.info("profiling index templates will not be installed or reinstalled"); + } + registry.get().setTemplatesEnabled(newValue); + indexManager.get().setTemplatesEnabled(newValue); } @Override @@ -114,7 +127,7 @@ public List getRestHandlers( @Override public List> getSettings() { return List.of( - PROFILING_ENABLED, + PROFILING_TEMPLATES_ENABLED, TransportGetProfilingAction.PROFILING_MAX_STACKTRACE_QUERY_SLICES, TransportGetProfilingAction.PROFILING_MAX_DETAIL_QUERY_SLICES, TransportGetProfilingAction.PROFILING_QUERY_REALTIME @@ -139,4 +152,10 @@ public static ExecutorBuilder responseExecutorBuilder() { public List> getActions() { return List.of(new ActionHandler<>(GetProfilingAction.INSTANCE, TransportGetProfilingAction.class)); } + + @Override + public void close() { + registry.get().close(); + indexManager.get().close(); + } } diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java index 873094f580535..0587ee5dbe08e 100644 --- a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java +++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java @@ -104,9 +104,9 @@ protected void doExecute(Task submitTask, GetProfilingRequest request, ActionLis log.debug("getResampledIndex took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms]."); searchEventGroupByStackTrace(client, request, resampledIndex, submitListener); }, e -> { - // Apart from profiling-events-all, indices are created lazily. In a relatively empty cluster it can happen - // that there are so few data that we need to resort to the full index. As this is an edge case we'd rather - // fail instead of prematurely checking for existence in all cases. + // All profiling-events data streams are created lazily. In a relatively empty cluster it can happen that there are so few + // data that we need to resort to the "full" events stream. As this is an edge case we'd rather fail instead of prematurely + // checking for existence in all cases. if (e instanceof IndexNotFoundException) { String missingIndex = ((IndexNotFoundException) e).getIndex().getName(); EventsIndex fullIndex = EventsIndex.FULL_INDEX; @@ -163,7 +163,15 @@ private void searchEventGroupByStackTrace( } else { submitListener.onResponse(responseBuilder.build()); } - }, submitListener::onFailure)); + }, e -> { + // Data streams are created lazily; if even the "full" index does not exist no data have been indexed yet. + if (e instanceof IndexNotFoundException) { + log.debug("Index [{}] does not exist. Returning empty response.", ((IndexNotFoundException) e).getIndex()); + submitListener.onResponse(responseBuilder.build()); + } else { + submitListener.onFailure(e); + } + })); } private void retrieveStackTraces( diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java new file mode 100644 index 0000000000000..fea912b1d5e97 --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexManagerTests.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.TestDiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ProfilingIndexManagerTests extends ESTestCase { + private final AtomicBoolean templatesCreated = new AtomicBoolean(); + private ProfilingIndexManager indexManager; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + templatesCreated.set(false); + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + indexManager = new ProfilingIndexManager(threadPool, client, clusterService) { + @Override + protected boolean isAllTemplatesCreated(ClusterChangedEvent event) { + return templatesCreated.get(); + } + }; + indexManager.setTemplatesEnabled(true); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + indexManager.clusterChanged(event); + } + + public void testThatMissingTemplatesDoesNothing() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + client.setVerifier((a, r, l) -> { + fail("if any templates are missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + indexManager.clusterChanged(event); + } + + public void testThatNonExistingIndicesAreAddedImmediately() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + templatesCreated.set(true); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyIndexInstalled(calledTimes, action, request, listener)); + indexManager.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(ProfilingIndexManager.INDICES_AND_ALIASES.size()))); + + calledTimes.set(0); + } + + public void testThatExistingIndicesAreNotCreatedTwice() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + templatesCreated.set(true); + + String existingIndex = randomFrom(ProfilingIndexManager.INDICES_AND_ALIASES.keySet()); + ClusterChangedEvent event = createClusterChangedEvent(List.of(existingIndex), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyIndexInstalled(calledTimes, action, request, listener)); + indexManager.clusterChanged(event); + // should not create the existing index + assertBusy(() -> assertThat(calledTimes.get(), equalTo(ProfilingIndexManager.INDICES_AND_ALIASES.size() - 1))); + + calledTimes.set(0); + } + + private ActionResponse verifyIndexInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof CreateIndexAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(CreateIndexAction.class)); + assertThat(request, instanceOf(CreateIndexRequest.class)); + assertNotNull(listener); + return new CreateIndexResponse(true, true, ((CreateIndexRequest) request).index()); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Iterable existingIndices, DiscoveryNodes nodes) { + ClusterState cs = createClusterState(Settings.EMPTY, existingIndices, nodes); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState(Settings nodeSettings, Iterable existingIndices, DiscoveryNodes nodes) { + Map indices = new HashMap<>(); + for (String index : existingIndices) { + IndexMetadata mockMetadata = mock(IndexMetadata.class); + when(mockMetadata.getIndex()).thenReturn(new Index(index, index)); + when(mockMetadata.getCompatibilityVersion()).thenReturn(Version.CURRENT); + indices.put(index, mockMetadata); + } + return ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().indices(indices).transientSettings(nodeSettings).build()) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } +} diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java new file mode 100644 index 0000000000000..3e8cbe05465ba --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/ProfilingIndexTemplateRegistryTests.java @@ -0,0 +1,340 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.node.TestDiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ilm.DeleteAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecycleAction; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ProfilingIndexTemplateRegistryTests extends ESTestCase { + private ProfilingIndexTemplateRegistry registry; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + registry = new ProfilingIndexTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY); + registry.setTemplatesEnabled(true); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); + registry.clusterChanged(event); + } + + public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + // component templates are already existing, just add composable templates + Map componentTemplates = new HashMap<>(); + for (String templateName : registry.getComponentTemplateConfigs().keySet()) { + componentTemplates.put(templateName, ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION); + } + ClusterChangedEvent event = createClusterChangedEvent(componentTemplates, Collections.emptyMap(), nodes); + + AtomicInteger calledTimes = new AtomicInteger(0); + + client.setVerifier((action, request, listener) -> verifyComposableTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComposableTemplateConfigs().size()))); + + calledTimes.set(0); + + // attempting to register the event multiple times as a race condition can yield this test flaky, namely: + // when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains + // might've not yet been updated to reflect that the first template registration was complete, so a second template registration + // will not be issued anymore, leaving calledTimes to 0 + assertBusy(() -> { + // now delete one template from the cluster state and lets retry + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); + registry.clusterChanged(newEvent); + assertThat(calledTimes.get(), greaterThan(1)); + }); + } + + public void testThatNonExistingPoliciesAreAddedImmediately() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutLifecycleAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutLifecycleAction.class)); + assertThat(request, instanceOf(PutLifecycleAction.Request.class)); + final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; + assertThat(putRequest.getPolicy().getName(), equalTo("profiling")); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + } + + public void testPolicyAlreadyExists() { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + + public void testPolicyAlreadyExistsButDiffers() throws IOException { + DiscoveryNode node = TestDiscoveryNode.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + Map policyMap = new HashMap<>(); + String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; + List policies = registry.getPolicyConfigs(); + assertThat(policies, hasSize(1)); + policies.forEach(p -> policyMap.put(p.getName(), p)); + + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + fail("if the policy already exists it should not be re-put"); + } else { + fail("client called with unexpected request: " + request.toString()); + } + return null; + }); + + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser( + XContentParserConfiguration.EMPTY.withRegistry( + new NamedXContentRegistry( + List.of( + new NamedXContentRegistry.Entry( + LifecycleAction.class, + new ParseField(DeleteAction.NAME), + DeleteAction::parse + ) + ) + ) + ), + policyStr + ) + ) { + LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); + policyMap.put(policies.get(0).getName(), different); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), policyMap, nodes); + registry.clusterChanged(event); + } + } + + private ActionResponse verifyComposableTemplateInstalled( + AtomicInteger calledTimes, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutComposableIndexTemplateAction.class)); + assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class)); + final PutComposableIndexTemplateAction.Request putRequest = ((PutComposableIndexTemplateAction.Request) request); + assertThat(putRequest.indexTemplate().version(), equalTo((long) ProfilingIndexTemplateRegistry.INDEX_TEMPLATE_VERSION)); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutIndexTemplateAction) { + // Ignore this, it's verified in another test + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + DiscoveryNodes nodes + ) { + return createClusterChangedEvent(existingComponentTemplates, existingComposableTemplates, Collections.emptyMap(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState( + Settings.EMPTY, + existingComponentTemplates, + existingComposableTemplates, + existingPolicies, + nodes + ); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComponentTemplates, + Map existingComposableTemplates, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map componentTemplates = new HashMap<>(); + for (Map.Entry template : existingComponentTemplates.entrySet()) { + ComponentTemplate mockTemplate = mock(ComponentTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + componentTemplates.put(template.getKey(), mockTemplate); + } + Map composableTemplates = new HashMap<>(); + for (Map.Entry template : existingComposableTemplates.entrySet()) { + ComposableIndexTemplate mockTemplate = mock(ComposableIndexTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + composableTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .componentTemplates(componentTemplates) + .indexTemplates(composableTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } +} diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java new file mode 100644 index 0000000000000..10282adf8a56f --- /dev/null +++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/VerifyingClient.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiler; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Assert; + +/** + * A client that delegates to a verifying function for action/request/listener + */ +public class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + Assert.fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 67a9cfcad7f50..3aca6e19cbe2a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -38,6 +38,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.LOGSTASH_MANAGEMENT_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.PROFILING_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ROLLUP_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.SEARCHABLE_SNAPSHOTS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; @@ -142,6 +143,7 @@ public static void switchUserBasedOnActionOriginAndExecute( case ENRICH_ORIGIN: case IDP_ORIGIN: case INGEST_ORIGIN: + case PROFILING_ORIGIN: case STACK_ORIGIN: case SEARCHABLE_SNAPSHOTS_ORIGIN: case LOGSTASH_MANAGEMENT_ORIGIN: