diff --git a/docs/changelog/100033.yaml b/docs/changelog/100033.yaml new file mode 100644 index 0000000000000..92ef6cd289fdc --- /dev/null +++ b/docs/changelog/100033.yaml @@ -0,0 +1,9 @@ +pr: 100033 +summary: "[Behavioral Analytics] Analytics collections use Data Stream Lifecycle (DSL)\ + \ instead of Index Lifecycle Management (ILM) for data retention management. Behavioral\ + \ analytics has traditionally used ILM to manage data retention. Starting with 8.12.0,\ + \ this will change. Analytics collections created prior to 8.12.0 will continue to use\ + \ their existing ILM policies, but new analytics collections will be managed using DSL." +area: Application +type: feature +issues: [ ] diff --git a/docs/reference/esql/functions/starts_with.asciidoc b/docs/reference/esql/functions/starts_with.asciidoc index b54caf362ac93..169abe607273f 100644 --- a/docs/reference/esql/functions/starts_with.asciidoc +++ b/docs/reference/esql/functions/starts_with.asciidoc @@ -1,5 +1,8 @@ [[esql-starts_with]] === `STARTS_WITH` +[.text-center] +image::esql/functions/signature/ends_with.svg[Embedded,opts=inline] + Returns a boolean that indicates whether a keyword string starts with another string: @@ -11,3 +14,7 @@ include::{esql-specs}/docs.csv-spec[tag=startsWith] |=== include::{esql-specs}/docs.csv-spec[tag=startsWith-result] |=== + +Supported types: + +include::types/starts_with.asciidoc[] diff --git a/docs/reference/esql/functions/trim.asciidoc b/docs/reference/esql/functions/trim.asciidoc index 0b6cf96ed1c5a..6743c5922ea21 100644 --- a/docs/reference/esql/functions/trim.asciidoc +++ b/docs/reference/esql/functions/trim.asciidoc @@ -1,5 +1,8 @@ [[esql-trim]] === `TRIM` +[.text-center] +image::esql/functions/signature/trim.svg[Embedded,opts=inline] + Removes leading and trailing whitespaces from strings. [source.merge.styled,esql] @@ -10,3 +13,7 @@ include::{esql-specs}/string.csv-spec[tag=trim] |=== include::{esql-specs}/string.csv-spec[tag=trim-result] |=== + +Supported types: + +include::types/trim.asciidoc[] diff --git a/docs/reference/ilm/actions/ilm-rollover.asciidoc b/docs/reference/ilm/actions/ilm-rollover.asciidoc index 9c4489bac1a94..4731986bd2559 100644 --- a/docs/reference/ilm/actions/ilm-rollover.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollover.asciidoc @@ -129,10 +129,10 @@ opt in to rolling over empty indices, by adding a `"min_docs": 0` condition. Thi disabled on a cluster-wide basis by setting `indices.lifecycle.rollover.only_if_has_documents` to `false`. -NOTE: The rollover action implicitly always rolls over a data stream or alias if one or more shards contain - 200000000 or more documents. Normally a shard will reach 50GB long before it reaches 200M documents, - but this isn't the case for space efficient data sets. Search performance will very likely suffer - if a shard contains more than 200M documents. This is the reason of the builtin limit. +IMPORTANT: The rollover action implicitly always rolls over a data stream or alias if one or more shards contain +200000000 or more documents. Normally a shard will reach 50GB long before it reaches 200M documents, +but this isn't the case for space efficient data sets. Search performance will very likely suffer +if a shard contains more than 200M documents. This is the reason of the builtin limit. [[ilm-rollover-ex]] ==== Example diff --git a/docs/reference/ilm/index-rollover.asciidoc b/docs/reference/ilm/index-rollover.asciidoc index a1616807c9ea6..5e6c4b89ba99f 100644 --- a/docs/reference/ilm/index-rollover.asciidoc +++ b/docs/reference/ilm/index-rollover.asciidoc @@ -51,3 +51,15 @@ TIP: Rolling over to a new index based on size, document count, or age is prefer to time-based rollovers. Rolling over at an arbitrary time often results in many small indices, which can have a negative impact on performance and resource usage. + +IMPORTANT: Empty indices will not be rolled over, even if they have an associated `max_age` that +would otherwise result in a roll over occurring. A policy can override this behavior, and explicitly +opt in to rolling over empty indices, by adding a `"min_docs": 0` condition. This can also be +disabled on a cluster-wide basis by setting `indices.lifecycle.rollover.only_if_has_documents` to +`false`. + +IMPORTANT: The rollover action implicitly always rolls over a data stream or alias if one or more shards contain +200000000 or more documents. Normally a shard will reach 50GB long before it reaches 200M documents, +but this isn't the case for space efficient data sets. Search performance will very likely suffer +if a shard contains more than 200M documents. This is the reason of the builtin limit. + diff --git a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc index cd256bfd966f5..79be6205a8c88 100644 --- a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc @@ -68,6 +68,11 @@ PUT _ilm/policy/my_policy <2> Delete the index 30 days after rollover ==== +IMPORTANT: The rollover action implicitly always rolls over a data stream or alias if one or more shards contain +200000000 or more documents. Normally a shard will reach 25GB long before it reaches 200M documents, +but this isn't the case for space efficient data sets. Search performance will very likely suffer +if a shard contains more than 200M documents. This is the reason of the builtin limit. + [discrete] [[apply-policy-template]] === Apply lifecycle policy with an index template diff --git a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc index e253ccda42376..303ad4da3ec7b 100644 --- a/docs/reference/query-rules/apis/get-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/get-query-ruleset.asciidoc @@ -52,9 +52,9 @@ PUT _query_rules/my-ruleset "type": "pinned", "criteria": [ { - "type": "exact", + "type": "contains", "metadata": "query_string", - "values": [ "marvel" ] + "values": [ "pugs", "puggles" ] } ], "actions": { @@ -69,9 +69,9 @@ PUT _query_rules/my-ruleset "type": "pinned", "criteria": [ { - "type": "exact", + "type": "fuzzy", "metadata": "query_string", - "values": [ "dc" ] + "values": [ "rescue dogs" ] } ], "actions": { @@ -117,9 +117,9 @@ A sample response: "type": "pinned", "criteria": [ { - "type": "exact", + "type": "contains", "metadata": "query_string", - "values": [ "marvel" ] + "values": [ "pugs", "puggles" ] } ], "actions": { @@ -134,9 +134,9 @@ A sample response: "type": "pinned", "criteria": [ { - "type": "exact", + "type": "fuzzy", "metadata": "query_string", - "values": [ "dc" ] + "values": [ "rescue dogs" ] } ], "actions": { diff --git a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc index e1a26a49cc525..6805201ce9d7c 100644 --- a/docs/reference/query-rules/apis/put-query-ruleset.asciidoc +++ b/docs/reference/query-rules/apis/put-query-ruleset.asciidoc @@ -22,29 +22,27 @@ Requires the `manage_search_query_rules` privilege. [role="child_attributes"] [[put-query-ruleset-request-body]] -(Required, object) -Contains parameters for a query ruleset: +(Required, object) Contains parameters for a query ruleset: ==== {api-request-body-title} + `rules`:: -(Required, array of objects) -The specific rules included in this query ruleset. +(Required, array of objects) The specific rules included in this query ruleset. Each rule must have the following information: -- `rule_id` (Required, string) - A unique identifier for this rule. -- `type` (Required, string) - The type of rule. At this time only `pinned` query rule types are allowed. -- `criteria` (Required, array of objects) - The criteria that must be met for the rule to be applied. If multiple criteria are specified for a rule, all criteria must be met for the rule to be applied. -- `actions` (Required, object) - The actions to take when the rule is matched. The format of this action depends on the rule type. +- `rule_id` (Required, string) A unique identifier for this rule. +- `type` (Required, string) The type of rule. +At this time only `pinned` query rule types are allowed. +- `criteria` (Required, array of objects) The criteria that must be met for the rule to be applied. +If multiple criteria are specified for a rule, all criteria must be met for the rule to be applied. +- `actions` (Required, object) The actions to take when the rule is matched. +The format of this action depends on the rule type. Criteria must have the following information: -- `type` (Required, string) - The type of criteria. The following criteria types are supported: +- `type` (Required, string) The type of criteria. +The following criteria types are supported: + -- - `exact` @@ -77,30 +75,32 @@ Only applicable for numerical values. - `always` Matches all queries, regardless of input. -- -- `metadata` (Optional, string) - The metadata field to match against. Required for all criteria types except `global`. -- `values` (Optional, array of strings) - The values to match against the metadata field. Only one value must match for the criteria to be met. Required for all criteria types except `global`. +- `metadata` (Optional, string) The metadata field to match against. +This metadata will be used to match against `match_criteria` sent in the <>. +Required for all criteria types except `global`. +- `values` (Optional, array of strings) The values to match against the metadata field. +Only one value must match for the criteria to be met. +Required for all criteria types except `global`. Actions depend on the rule type. For `pinned` rules, actions follow the format specified by the <>. The following actions are allowed: -- `ids` (Optional, array of strings) - The The unique <> of the documents to pin. - Only one of `ids` or `docs` may be specified, and at least one must be specified. -- `docs` (Optional, array of objects) - The documents to pin. Only one of `ids` or `docs` may be specified, and at least one must be specified. - You can specify the following attributes for each document: +- `ids` (Optional, array of strings) The unique <> of the documents to pin. +Only one of `ids` or `docs` may be specified, and at least one must be specified. +- `docs` (Optional, array of objects) The documents to pin. +Only one of `ids` or `docs` may be specified, and at least one must be specified. +You can specify the following attributes for each document: + -- -- `_index` (Required, string) - The index of the document to pin. -- `_id` (Required, string) - The unique <>. +- `_index` (Required, string) The index of the document to pin. +- `_id` (Required, string) The unique <>. -- -IMPORTANT: Due to limitations within <>, you can only pin documents using `ids` or `docs`, but cannot use both in single rule. It is advised to use one or the other in query rulesets, to avoid errors. Additionally, pinned queries have a maximum limit of 100 pinned hits. If multiple matching rules pin more than 100 documents, only the first 100 documents are pinned in the order they are specified in the ruleset. +IMPORTANT: Due to limitations within <>, you can only pin documents using `ids` or `docs`, but cannot use both in single rule. +It is advised to use one or the other in query rulesets, to avoid errors. +Additionally, pinned queries have a maximum limit of 100 pinned hits. +If multiple matching rules pin more than 100 documents, only the first 100 documents are pinned in the order they are specified in the ruleset. [[put-query-ruleset-example]] ==== {api-examples-title} @@ -109,8 +109,8 @@ The following example creates a new query ruleset called `my-ruleset`. Two rules are associated with `my-ruleset`: -- `my-rule1` will pin documents with IDs `id1` and `id2` when `user.query` exactly matches `marvel` _or_ `dc` **and** `user.country` exactly matches `us`. -- `my-rule2` will pin documents from different, specified indices with IDs `id3` and `id4` when the `query_string` fuzzily matches `comic`. +- `my-rule1` will pin documents with IDs `id1` and `id2` when `user_query` contains `pugs` _or_ `puggles` **and** `user_country` exactly matches `us`. +- `my-rule2` will pin documents from different, specified indices with IDs `id3` and `id4` when the `query_string` fuzzily matches `rescue dogs`. [source,console] ---- @@ -123,12 +123,12 @@ PUT _query_rules/my-ruleset "criteria": [ { "type": "contains", - "metadata": "user.query", - "values": [ "marvel", "dc" ] + "metadata": "user_query", + "values": [ "pugs", "puggles" ] }, { "type": "exact", - "metadata": "user.country", + "metadata": "user_country", "values": [ "us" ] } ], @@ -145,8 +145,8 @@ PUT _query_rules/my-ruleset "criteria": [ { "type": "fuzzy", - "metadata": "query_string", - "values": [ "comic" ] + "metadata": "user_query", + "values": [ "rescue dogs" ] } ], "actions": { diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 08d64e2b9353b..f3edc7a90646a 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -41,6 +41,16 @@ excludeList.add('aggregations/filter/Standard queries get cached') excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') +// These tests check setting validations in the desired_node API. +// Validation (and associated tests) are supposed to be skipped/have +// different behaviour for versions before and after 8.10 but mixed +// cluster tests may not respect that - see the comment above. +excludeList.add('cluster.desired_nodes/10_basic/Test settings are validated') +excludeList.add('cluster.desired_nodes/10_basic/Test unknown settings are forbidden in known versions') +excludeList.add('cluster.desired_nodes/10_basic/Test unknown settings are allowed in future versions') +excludeList.add('cluster.desired_nodes/10_basic/Test some settings can be overridden') +excludeList.add('cluster.desired_nodes/20_dry_run/Test validation works for dry run updates') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml index fa7a5ae1449a9..cb897c918305e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.9.99" + reason: "position metric introduced in 8.8.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index e4963832ddce3..3d297e2181970 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -424,6 +424,10 @@ nested fields: --- "Synthetic source": + - skip: + version: " - 8.9.99" + reason: Synthetic source shows up in the mapping in 8.10 + - do: indices.create: index: tsdb-synthetic diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java index 4253eeb3b057e..1f31b2155846e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/settings/ClusterSettingsUpdateWithFaultyMasterIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.NetworkDisruption; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import java.util.Collection; @@ -36,6 +37,10 @@ protected Collection> nodePlugins() { return List.of(BlockingClusterSettingTestPlugin.class, MockTransportService.TestPlugin.class); } + @TestLogging( + reason = "https://github.com/elastic/elasticsearch/issues/98918", + value = "org.elasticsearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction:TRACE" + ) public void testClusterSettingsUpdateNotAcknowledged() throws Exception { final var nodes = internalCluster().startMasterOnlyNodes(3); final String masterNode = internalCluster().getMasterName(); @@ -52,7 +57,7 @@ public void testClusterSettingsUpdateNotAcknowledged() throws Exception { ); internalCluster().setDisruptionScheme(networkDisruption); - logger.debug("--> updating cluster settings"); + logger.info("--> updating cluster settings"); var future = client(masterNode).admin() .cluster() .prepareUpdateSettings() @@ -60,18 +65,18 @@ public void testClusterSettingsUpdateNotAcknowledged() throws Exception { .setMasterNodeTimeout(TimeValue.timeValueMillis(10L)) .execute(); - logger.debug("--> waiting for cluster state update to be blocked"); - BlockingClusterSettingTestPlugin.blockLatch.await(); + logger.info("--> waiting for cluster state update to be blocked"); + safeAwait(BlockingClusterSettingTestPlugin.blockLatch); - logger.debug("--> isolating master eligible node [{}] from other nodes", blockedNode); + logger.info("--> isolating master eligible node [{}] from other nodes", blockedNode); networkDisruption.startDisrupting(); - logger.debug("--> unblocking cluster state update"); + logger.info("--> unblocking cluster state update"); BlockingClusterSettingTestPlugin.releaseLatch.countDown(); assertThat("--> cluster settings update should not be acknowledged", future.get().isAcknowledged(), equalTo(false)); - logger.debug("--> stop network disruption"); + logger.info("--> stop network disruption"); networkDisruption.stopDisrupting(); ensureStableCluster(3); } @@ -86,11 +91,13 @@ public static class BlockingClusterSettingTestPlugin extends Plugin { public static final Setting TEST_BLOCKING_SETTING = Setting.boolSetting("cluster.test.blocking_setting", false, value -> { if (blockOnce.compareAndSet(false, true)) { - logger.debug("--> setting validation is now blocking cluster state update"); + logger.info("--> setting validation is now blocking cluster state update"); blockLatch.countDown(); - logger.debug("--> setting validation is now waiting for release"); + logger.info("--> setting validation is now waiting for release"); safeAwait(releaseLatch); - logger.debug("--> setting validation is done"); + logger.info("--> setting validation is done"); + } else { + logger.info("--> setting validation was blocked before"); } }, Setting.Property.NodeScope, Setting.Property.Dynamic); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index bdd7fbd999a7b..5765b0fc4b99a 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -383,6 +383,7 @@ private void snapshot( SnapshotIndexCommit snapshotIndexCommit = null; try { snapshotIndexCommit = new SnapshotIndexCommit(indexShard.acquireIndexCommitForSnapshot()); + final var shardStateId = getShardStateId(indexShard, snapshotIndexCommit.indexCommit()); // not aborted so indexCommit() ok snapshotStatus.addAbortListener(makeAbortListener(indexShard.shardId(), snapshot, snapshotIndexCommit)); snapshotStatus.ensureNotAborted(); repository.snapshotShard( @@ -392,7 +393,7 @@ private void snapshot( snapshot.getSnapshotId(), indexId, snapshotIndexCommit, - getShardStateId(indexShard, snapshotIndexCommit.indexCommit()), + shardStateId, snapshotStatus, version, entryStartTime, diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java index 803a45ad13b07..31baba05c3b09 100644 --- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java +++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java @@ -1364,7 +1364,6 @@ public void testCancelViaTasksAPI() throws Exception { assertThat(json, matchesRegex(".*task (was)?\s*cancelled.*")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99519") public void testCancelViaAsyncSearchDelete() throws Exception { Map testClusterInfo = setupTwoClusters(); String localIndex = (String) testClusterInfo.get("local.index"); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-default_policy.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-default_policy.json similarity index 70% rename from x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-default_policy.json rename to x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-default_policy.json index 5256d70496f7e..6a2a6ce79bbfb 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-default_policy.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-default_policy.json @@ -25,13 +25,13 @@ }, "delete": { "min_age": "180d", - "actions":{ + "actions": { "delete": {} } } }, "_meta": { - "description": "Built-in policy applied by default to behavioral analytics event data streams.", + "description": "DEPRECATED: Built-in policy applied by default to behavioral analytics event data streams. This should be deleted after all ILM-backed Behavioral Analytics indices are transitioned to DSL.", "managed": true } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-settings.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-settings.json index df27c6f77fd44..9aa9731be6524 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/analytics/behavioral_analytics-events-settings.json @@ -3,7 +3,8 @@ "settings": { "index": { "lifecycle": { - "name": "behavioral_analytics-events-default_policy" + "name": "behavioral_analytics-events-default_policy", + "prefer_ilm": false }, "codec": "best_compression", "number_of_shards": 1, @@ -15,6 +16,9 @@ "order": ["asc", "asc"] } } + }, + "lifecycle": { + "data_retention": "180d" } }, "_meta": { diff --git a/x-pack/plugin/ent-search/qa/build.gradle b/x-pack/plugin/ent-search/qa/build.gradle index e69de29bb2d1d..dbdb4b6b6bf32 100644 --- a/x-pack/plugin/ent-search/qa/build.gradle +++ b/x-pack/plugin/ent-search/qa/build.gradle @@ -0,0 +1,3 @@ +subprojects { + group = "org.elasticsearch.ent-search.qa" +} diff --git a/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle b/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle new file mode 100644 index 0000000000000..f92f1b6223fcc --- /dev/null +++ b/x-pack/plugin/ent-search/qa/full-cluster-restart/build.gradle @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.bwc-test' + + +dependencies { + testImplementation project(path: xpackModule('ent-search')) + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest")) +} + +assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8: + "If we are targeting a branch other than 8, we should enable migration tests" + +BuildParams.bwcVersions.withWireCompatible(v -> v.after("8.8.0")) { bwcVersion, baseName -> + tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) { + usesBwcDistribution(bwcVersion) + systemProperty("tests.old_cluster_version", bwcVersion) + } +} + + +testClusters.configureEach { + testDistribution = 'DEFAULT' + numberOfNodes = 1 + setting 'xpack.license.self_generated.type', 'trial' +} diff --git a/x-pack/plugin/ent-search/qa/full-cluster-restart/javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java b/x-pack/plugin/ent-search/qa/full-cluster-restart/javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java new file mode 100644 index 0000000000000..30098871fb805 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/full-cluster-restart/javaRestTest/java/org/elasticsearch/xpack/application/FullClusterRestartIT.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.xpack.application; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; +import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.List; + +public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase { + + private static final Version DSL_DEFAULT_RETENTION_VERSION = V_8_12_0; + + // Legacy name we used for ILM policy configuration in versions prior to 8.12.0. + private static final String EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME = "behavioral_analytics-events-default_policy"; + + @ClassRule + public static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("xpack.security.enabled", "false") + .module("x-pack-ent-search") + .build(); + + public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + public void testBehavioralAnalyticsDataRetention() throws Exception { + + assumeTrue( + "Data retention changed by default to DSL in " + DSL_DEFAULT_RETENTION_VERSION, + getOldClusterTestVersion().before(DSL_DEFAULT_RETENTION_VERSION) + ); + + String legacyAnalyticsCollectionName = "oldstuff"; + String newAnalyticsCollectionName = "newstuff"; + + if (isRunningAgainstOldCluster()) { + // Create an analytics collection + Request legacyPutRequest = new Request("PUT", "_application/analytics/" + legacyAnalyticsCollectionName); + assertOK(client().performRequest(legacyPutRequest)); + + // Validate that ILM lifecycle is in place + assertBusy(() -> assertLegacyDataRetentionPolicy(legacyAnalyticsCollectionName)); + } else { + // Create a new analytics collection + Request putRequest = new Request("PUT", "_application/analytics/" + newAnalyticsCollectionName); + assertOK(client().performRequest(putRequest)); + + // Validate that NO ILM lifecycle is in place and we are using DLS instead. + assertBusy(() -> assertDslDataRetention(newAnalyticsCollectionName)); + + // Validate that the existing analytics collection created with an older version is still using ILM + assertBusy(() -> assertLegacyDataRetentionPolicy(legacyAnalyticsCollectionName)); + } + } + + private void assertLegacyDataRetentionPolicy(String analyticsCollectionName) throws IOException { + String dataStreamName = "behavioral_analytics-events-" + analyticsCollectionName; + Request getDataStreamRequest = new Request("GET", "_data_stream/" + dataStreamName); + Response response = client().performRequest(getDataStreamRequest); + assertOK(response); + ObjectPath dataStream = ObjectPath.createFromResponse(response); + String pathToIlmPolicy = "data_streams.0.ilm_policy"; + assertNotNull(dataStream.evaluate(pathToIlmPolicy)); + assertEquals(EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME, dataStream.evaluate(pathToIlmPolicy)); + + Request policyRequest = new Request("GET", "_ilm/policy/" + EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME); + ObjectPath policy = ObjectPath.createFromResponse(client().performRequest(policyRequest)); + assertNotNull(policy.evaluate(EVENT_DATA_STREAM_LEGACY_ILM_POLICY_NAME)); + } + + private void assertDslDataRetention(String analyticsCollectionName) throws IOException { + String dataStreamName = "behavioral_analytics-events-" + analyticsCollectionName; + Request getDataStreamRequest = new Request("GET", "_data_stream/" + dataStreamName); + Response response = client().performRequest(getDataStreamRequest); + assertOK(response); + ObjectPath dataStreamResponse = ObjectPath.createFromResponse(response); + + List dataStreams = dataStreamResponse.evaluate("data_streams"); + boolean evaluatedNewDataStream = false; + for (Object dataStreamObj : dataStreams) { + ObjectPath dataStream = new ObjectPath(dataStreamObj); + if (dataStreamName.equals(dataStream.evaluate("name"))) { + assertNull(dataStream.evaluate("ilm_policy")); + assertEquals(true, dataStream.evaluate("lifecycle.enabled")); + assertEquals("180d", dataStream.evaluate("lifecycle.data_retention")); + evaluatedNewDataStream = true; + } + } + assertTrue(evaluatedNewDataStream); + + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java index 4d4804ee7c5a5..0ecdca59c0f33 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java @@ -17,11 +17,9 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; -import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; import java.io.IOException; import java.util.HashMap; @@ -36,18 +34,11 @@ public class AnalyticsTemplateRegistry extends IndexTemplateRegistry { - // This registry requires all nodes to be at least 8.8.0 - static final Version MIN_NODE_VERSION = Version.V_8_8_0; + // This registry requires all nodes to be at least 8.12.0 + static final Version MIN_NODE_VERSION = Version.V_8_12_0; // This number must be incremented when we make changes to built-in templates. - static final int REGISTRY_VERSION = 2; - - // ILM Policies configuration - static final String EVENT_DATA_STREAM_ILM_POLICY_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "default_policy"; - - static final List LIFECYCLE_POLICIES_CONFIG = List.of( - new LifecyclePolicyConfig(EVENT_DATA_STREAM_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + EVENT_DATA_STREAM_ILM_POLICY_NAME + ".json") - ); + static final int REGISTRY_VERSION = 3; // Index template components configuration static final String EVENT_DATA_STREAM_SETTINGS_COMPONENT_NAME = EVENT_DATA_STREAM_INDEX_PREFIX + "settings"; @@ -125,11 +116,6 @@ protected String getOrigin() { return ENT_SEARCH_ORIGIN; } - @Override - protected List getLifecycleConfigs() { - return LIFECYCLE_POLICIES_CONFIG; - } - @Override protected Map getComponentTemplateConfigs() { return COMPONENT_TEMPLATES; @@ -140,12 +126,6 @@ protected Map getComposableTemplateConfigs() { return COMPOSABLE_INDEX_TEMPLATES; } - // overriden to be visible in tests - @Override - protected List getLifecyclePolicies() { - return super.getLifecyclePolicies(); - } - @Override protected boolean requiresMasterNode() { // We are using the composable index template and component APIs, diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistryTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistryTests.java index 9bae2d020fba1..d1e0e23ee3230 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistryTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistryTests.java @@ -40,13 +40,8 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; -import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; import org.elasticsearch.xpack.core.ilm.OperationMode; @@ -54,7 +49,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -153,132 +147,6 @@ public void testThatNonExistingComponentTemplatesAreAddedImmediately() throws Ex }); } - public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - AtomicInteger calledTimes = new AtomicInteger(0); - client.setVerifier((action, request, listener) -> { - if (action instanceof PutPipelineAction) { - calledTimes.incrementAndGet(); - return AcknowledgedResponse.TRUE; - } - if (action instanceof PutLifecycleAction) { - calledTimes.incrementAndGet(); - assertThat(action, instanceOf(PutLifecycleAction.class)); - assertThat(request, instanceOf(PutLifecycleAction.Request.class)); - final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; - assertThat(putRequest.getPolicy().getName(), equalTo(AnalyticsTemplateRegistry.EVENT_DATA_STREAM_ILM_POLICY_NAME)); - assertNotNull(listener); - return AcknowledgedResponse.TRUE; - } else if (action instanceof PutComponentTemplateAction) { - // Ignore this, it's verified in another test - return new AnalyticsTemplateRegistryTests.TestPutIndexTemplateResponse(true); - } else if (action instanceof PutComposableIndexTemplateAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } else { - fail("client called with unexpected request: " + request.toString()); - return null; - } - }); - - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), Collections.emptyMap(), nodes); - registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(2))); - } - - public void testPolicyAlreadyExists() { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - Map policyMap = new HashMap<>(); - List policies = registry.getLifecyclePolicies(); - assertThat(policies, hasSize(1)); - policies.forEach(p -> policyMap.put(p.getName(), p)); - - client.setVerifier((action, request, listener) -> { - if (action instanceof PutPipelineAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } - if (action instanceof PutComponentTemplateAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } else if (action instanceof PutLifecycleAction) { - fail("if the policy already exists it should not be re-put"); - } else { - fail("client called with unexpected request: " + request.toString()); - } - return null; - }); - - ClusterChangedEvent event = createClusterChangedEvent( - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptyMap(), - policyMap, - nodes - ); - registry.clusterChanged(event); - } - - public void testPolicyAlreadyExistsButDiffers() throws IOException { - DiscoveryNode node = DiscoveryNodeUtils.create("node"); - DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - - Map policyMap = new HashMap<>(); - String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}"; - List policies = registry.getLifecyclePolicies(); - assertThat(policies, hasSize(1)); - policies.forEach(p -> policyMap.put(p.getName(), p)); - - client.setVerifier((action, request, listener) -> { - if (action instanceof PutPipelineAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } - if (action instanceof PutComponentTemplateAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; - } else if (action instanceof PutLifecycleAction) { - fail("if the policy already exists it should not be re-put"); - } else { - fail("client called with unexpected request: " + request.toString()); - } - return null; - }); - - try ( - XContentParser parser = XContentType.JSON.xContent() - .createParser( - XContentParserConfiguration.EMPTY.withRegistry( - new NamedXContentRegistry( - List.of( - new NamedXContentRegistry.Entry( - LifecycleAction.class, - new ParseField(DeleteAction.NAME), - DeleteAction::parse - ) - ) - ) - ), - policyStr - ) - ) { - LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName()); - policyMap.put(policies.get(0).getName(), different); - ClusterChangedEvent event = createClusterChangedEvent( - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptyMap(), - policyMap, - nodes - ); - registry.clusterChanged(event); - } - } - public void testThatVersionedOldComponentTemplatesAreUpgraded() throws Exception { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); @@ -334,8 +202,8 @@ public void testSameOrHigherVersionComponentTemplateNotUpgraded() { fail("template should not have been re-installed"); return null; } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; + fail("As of 8.12.0 we no longer put an ILM lifecycle and instead rely on DSL for analytics datastreams."); + return null; } else if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; @@ -390,8 +258,7 @@ public void testThatNonExistingPipelinesAreAddedImmediately() throws Exception { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; + fail("As of 8.12.0 we no longer put an ILM lifecycle and instead rely on DSL for analytics datastreams."); } else if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; @@ -475,8 +342,8 @@ private ActionResponse verifyComposableTemplateInstalled( // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; + fail("As of 8.12.0 we no longer put an ILM lifecycle and instead rely on DSL for analytics datastreams."); + return null; } else if (action instanceof PutComposableIndexTemplateAction) { calledTimes.incrementAndGet(); assertThat(action, instanceOf(PutComposableIndexTemplateAction.class)); @@ -512,8 +379,8 @@ private ActionResponse verifyComponentTemplateInstalled( assertNotNull(listener); return new TestPutIndexTemplateResponse(true); } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return AcknowledgedResponse.TRUE; + fail("As of 8.12.0 we no longer put an ILM lifecycle and instead rely on DSL for analytics datastreams."); + return null; } else if (action instanceof PutComposableIndexTemplateAction) { // Ignore this, it's verified in another test return AcknowledgedResponse.TRUE; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java index 2caf338d2a3c7..ca9a99f2d96f5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java @@ -164,6 +164,8 @@ public void clusterChanged(ClusterChangedEvent event) { } if (eventStateMinTransportVersionIsBeforeDistributedModelAllocationTransportVersion(event)) { + logger.trace("min transport version is before assignment change on " + event.state().nodes().getAllNodes().size() + " nodes"); + // we should not try to rebalance assignments while there may be nodes running on a version // prior to introducing distributed model allocation. // But we should remove routing to removed or shutting down nodes. @@ -238,6 +240,7 @@ public void onFailure(Exception e) { } private void removeRoutingToRemovedOrShuttingDownNodes(ClusterChangedEvent event) { + logger.trace("remove routing to removed or shutting down nodes "); if (areAssignedNodesRemoved(event)) { submitUnbatchedTask("removing routing entries for removed or shutting down nodes", new ClusterStateUpdateTask() { @Override @@ -282,6 +285,7 @@ static boolean areAssignedNodesRemoved(ClusterChangedEvent event) { // Visible for testing static ClusterState removeRoutingToUnassignableNodes(ClusterState currentState) { + logger.trace("remove routing to unassignable nodes"); Set assignableNodes = getAssignableNodes(currentState).stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); TrainedModelAssignmentMetadata metadata = TrainedModelAssignmentMetadata.fromState(currentState); TrainedModelAssignmentMetadata.Builder builder = TrainedModelAssignmentMetadata.builder(currentState); @@ -431,6 +435,7 @@ public void createNewModelAssignment( } public void setModelAssignmentToStopping(String modelId, ActionListener listener) { + logger.trace("set to stopping"); submitUnbatchedTask("set model assignment stopping", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { @@ -450,6 +455,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) } public void removeModelAssignment(String deploymentId, ActionListener listener) { + logger.trace("remove model assignments"); submitUnbatchedTask("delete model deployment assignment", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { @@ -486,6 +492,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) // Used by the reset action directly public void removeAllModelAssignments(ActionListener listener) { + logger.trace("remove all assignments"); submitUnbatchedTask("delete all model assignments", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { @@ -518,9 +525,11 @@ private static ClusterState forceUpdate(ClusterState currentState, TrainedModelA logger.debug(() -> format("updated assignments: %s", modelAssignments.build())); Metadata.Builder metadata = Metadata.builder(currentState.metadata()); if (currentState.getMinTransportVersion().onOrAfter(RENAME_ALLOCATION_TO_ASSIGNMENT_TRANSPORT_VERSION)) { + logger.trace("putting custom new name"); metadata.putCustom(TrainedModelAssignmentMetadata.NAME, modelAssignments.build()) .removeCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME); } else { + logger.trace("putting custom old name"); metadata.putCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME, modelAssignments.buildOld()); } return ClusterState.builder(currentState).metadata(metadata).build(); @@ -616,6 +625,7 @@ ClusterState stopPlatformSpecificModelsInHeterogeneousClusters( modelToAdd.get().getModelId(), mlNodesArchitectures ); + logger.info(reasonToStop); updatedState = callSetToStopping(reasonToStop, modelToAdd.get().getDeploymentId(), clusterState); } return updatedState; diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index b9fbf0b6b1f03..87d605d29fa86 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -73,7 +73,7 @@ public void setUpLogging() throws IOException { { "persistent": { "logger.org.elasticsearch.xpack.ml.inference": "TRACE", - "logger.org.elasticsearch.xpack.ml.inference.assignments": "DEBUG", + "logger.org.elasticsearch.xpack.ml.inference.assignments": "TRACE", "logger.org.elasticsearch.xpack.ml.process": "DEBUG", "logger.org.elasticsearch.xpack.ml.action": "TRACE" } @@ -97,6 +97,7 @@ public void removeLogging() throws IOException { client().performRequest(request); } + @AwaitsFix(bugUrl = "mute to try and reproduce https://github.com/elastic/elasticsearch/issues/100379") public void testTrainedModelDeployment() throws Exception { assumeTrue("NLP model deployments added in 8.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_0_0));