diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java index f7d2b26e069e1..bbf411dbf04fa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java @@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider { "org.elasticsearch.nativeaccess", "org.elasticsearch.plugin", "org.elasticsearch.plugin.analysis", - "org.elasticsearch.pluginclassloader", "org.elasticsearch.securesm", "org.elasticsearch.server", "org.elasticsearch.simdvec", diff --git a/distribution/tools/entitlement-agent/README.md b/distribution/tools/entitlement-agent/README.md new file mode 100644 index 0000000000000..ff40651706a7f --- /dev/null +++ b/distribution/tools/entitlement-agent/README.md @@ -0,0 +1,10 @@ +### Entitlement Agent + +This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers. + +This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module. + +It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`. diff --git a/distribution/tools/entitlement-agent/build.gradle b/distribution/tools/entitlement-agent/build.gradle new file mode 100644 index 0000000000000..56e2ffac53fd7 --- /dev/null +++ b/distribution/tools/entitlement-agent/build.gradle @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +apply plugin: 'elasticsearch.build' + +configurations { + entitlementRuntime +} + +dependencies { + entitlementRuntime project(":libs:elasticsearch-entitlement-runtime") + implementation project(":libs:elasticsearch-entitlement-runtime") + testImplementation project(":test:framework") +} + +tasks.named('test').configure { + dependsOn('jar') + jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}" +} + +tasks.named('jar').configure { + manifest { + attributes( + 'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent' + , 'Can-Retransform-Classes': 'true' + ) + } +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + diff --git a/libs/plugin-classloader/src/main/java/module-info.java b/distribution/tools/entitlement-agent/src/main/java/module-info.java similarity index 78% rename from libs/plugin-classloader/src/main/java/module-info.java rename to distribution/tools/entitlement-agent/src/main/java/module-info.java index 90549c5c4a01b..df6fc154fc67f 100644 --- a/libs/plugin-classloader/src/main/java/module-info.java +++ b/distribution/tools/entitlement-agent/src/main/java/module-info.java @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -module org.elasticsearch.pluginclassloader { - exports org.elasticsearch.plugins.loader; +module org.elasticsearch.entitlement.agent { + requires java.instrument; + requires org.elasticsearch.entitlement.runtime; } diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java b/distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java similarity index 53% rename from server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java rename to distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java index 1a9618dd59f4d..b843e42f4a03e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ack/AckedRequest.java +++ b/distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java @@ -7,22 +7,15 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.cluster.ack; +package org.elasticsearch.entitlement.agent; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.entitlement.runtime.api.EntitlementChecks; -/** - * Identifies a cluster state update request with acknowledgement support - */ -public interface AckedRequest { +import java.lang.instrument.Instrumentation; - /** - * Returns the acknowledgement timeout - */ - TimeValue ackTimeout(); +public class EntitlementAgent { - /** - * Returns the timeout for the request to be completed on the master node - */ - TimeValue masterNodeTimeout(); + public static void premain(String agentArgs, Instrumentation inst) throws Exception { + EntitlementChecks.setAgentBooted(); + } } diff --git a/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java new file mode 100644 index 0000000000000..3927465570c98 --- /dev/null +++ b/distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.agent; + +import org.elasticsearch.entitlement.runtime.api.EntitlementChecks; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ESTestCase.WithoutSecurityManager; + +/** + * This is an end-to-end test that runs with the javaagent installed. + * It should exhaustively test every instrumented method to make sure it passes with the entitlement + * and fails without it. + * See {@code build.gradle} for how we set the command line arguments for this test. + */ +@WithoutSecurityManager +public class EntitlementAgentTests extends ESTestCase { + + public void testAgentBooted() { + assertTrue(EntitlementChecks.isAgentBooted()); + } + +} diff --git a/docs/changelog/112768.yaml b/docs/changelog/112768.yaml new file mode 100644 index 0000000000000..13d5b8eaae38f --- /dev/null +++ b/docs/changelog/112768.yaml @@ -0,0 +1,5 @@ +pr: 112768 +summary: Deduplicate Kuromoji User Dictionary +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/113102.yaml b/docs/changelog/113102.yaml new file mode 100644 index 0000000000000..ea9022e634caf --- /dev/null +++ b/docs/changelog/113102.yaml @@ -0,0 +1,5 @@ +pr: 113102 +summary: Trigger merges after recovery +area: Recovery +type: enhancement +issues: [] diff --git a/docs/changelog/113103.yaml b/docs/changelog/113103.yaml new file mode 100644 index 0000000000000..2ed98e0907bae --- /dev/null +++ b/docs/changelog/113103.yaml @@ -0,0 +1,6 @@ +pr: 113103 +summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological" +area: ES|QL +type: bug +issues: + - 112482 diff --git a/docs/changelog/113123.yaml b/docs/changelog/113123.yaml new file mode 100644 index 0000000000000..43008eaa80f43 --- /dev/null +++ b/docs/changelog/113123.yaml @@ -0,0 +1,6 @@ +pr: 113123 +summary: "ES|QL: Skip CASE function from `InferIsNotNull` rule checks" +area: ES|QL +type: bug +issues: + - 112704 diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index b1d1d5a751057..e8380ce0aca17 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -133,6 +133,11 @@ unknown words. It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -221,7 +226,8 @@ PUT kuromoji_sample "type": "kuromoji_tokenizer", "mode": "extended", "discard_punctuation": "false", - "user_dictionary": "userdict_ja.txt" + "user_dictionary": "userdict_ja.txt", + "lenient": "true" } }, "analyzer": { diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc index 1a3153fa3bea5..e7855f94758e1 100644 --- a/docs/plugins/analysis-nori.asciidoc +++ b/docs/plugins/analysis-nori.asciidoc @@ -58,6 +58,11 @@ It can be set to: Whether punctuation should be discarded from the output. Defaults to `true`. +`lenient`:: + + Whether the `user_dictionary` should be deduplicated on the provided `text`. + False by default causing duplicates to generate an error. + `user_dictionary`:: + -- @@ -104,7 +109,8 @@ PUT nori_sample "type": "nori_tokenizer", "decompound_mode": "mixed", "discard_punctuation": "false", - "user_dictionary": "userdict_ko.txt" + "user_dictionary": "userdict_ko.txt", + "lenient": "true" } }, "analyzer": { @@ -299,7 +305,6 @@ Which responds with: } -------------------------------------------------- - [[analysis-nori-speech]] ==== `nori_part_of_speech` token filter diff --git a/docs/reference/esql/functions/examples/date_diff.asciidoc b/docs/reference/esql/functions/examples/date_diff.asciidoc index f85bdf480c1c3..f75add7b80501 100644 --- a/docs/reference/esql/functions/examples/date_diff.asciidoc +++ b/docs/reference/esql/functions/examples/date_diff.asciidoc @@ -1,6 +1,6 @@ // This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. -*Example* +*Examples* [source.merge.styled,esql] ---- @@ -10,4 +10,15 @@ include::{esql-specs}/date.csv-spec[tag=docsDateDiff] |=== include::{esql-specs}/date.csv-spec[tag=docsDateDiff-result] |=== +When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. +To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + +[source.merge.styled,esql] +---- +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/date.csv-spec[tag=evalDateDiffYearForDocs-result] +|=== diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index 4ba0d598a689c..f4c4de53f72a3 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -54,7 +54,8 @@ } ], "examples" : [ - "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)" + "ROW date1 = TO_DATETIME(\"2023-12-02T11:00:00.000Z\"), date2 = TO_DATETIME(\"2023-12-02T11:00:00.001Z\")\n| EVAL dd_ms = DATE_DIFF(\"microseconds\", date1, date2)", + "ROW end_23=\"2023-12-31T23:59:59.999Z\"::DATETIME,\n start_24=\"2024-01-01T00:00:00.000Z\"::DATETIME,\n end_24=\"2024-12-31T23:59:59.999\"::DATETIME\n| EVAL end23_to_start24=DATE_DIFF(\"year\", end_23, start_24)\n| EVAL end23_to_end24=DATE_DIFF(\"year\", end_23, end_24)\n| EVAL start_to_end_24=DATE_DIFF(\"year\", start_24, end_24)" ], "preview" : false } diff --git a/docs/reference/esql/functions/kibana/docs/mv_avg.md b/docs/reference/esql/functions/kibana/docs/mv_avg.md index c3d7e5423f724..c5163f36129bf 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_avg.md +++ b/docs/reference/esql/functions/kibana/docs/mv_avg.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_AVG -Converts a multivalued field into a single valued field containing the average of all of the values. +Converts a multivalued field into a single valued field containing the average of all the values. ``` ROW a=[3, 5, 1, 6] diff --git a/docs/reference/esql/functions/kibana/docs/mv_sum.md b/docs/reference/esql/functions/kibana/docs/mv_sum.md index 16285d3c7229b..987017b34b743 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_sum.md +++ b/docs/reference/esql/functions/kibana/docs/mv_sum.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MV_SUM -Converts a multivalued field into a single valued field containing the sum of all of the values. +Converts a multivalued field into a single valued field containing the sum of all the values. ``` ROW a=[3, 5, 6] diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 572cad591fba6..efa0c78b8356f 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -179,6 +179,7 @@ PUT _inference/text_embedding/my-e5-model "min_number_of_allocations": 3, "max_number_of_allocations": 10 }, + "num_threads": 1, "model_id": ".multilingual-e5-small" } } diff --git a/docs/reference/inference/service-elser.asciidoc b/docs/reference/inference/service-elser.asciidoc index fdce94901984b..c7217f38d459b 100644 --- a/docs/reference/inference/service-elser.asciidoc +++ b/docs/reference/inference/service-elser.asciidoc @@ -147,7 +147,8 @@ PUT _inference/sparse_embedding/my-elser-model "enabled": true, "min_number_of_allocations": 3, "max_number_of_allocations": 10 - } + }, + "num_threads": 1 } } ------------------------------------------------------------ diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index 08dd7ab7f4470..399cf29d4dd12 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -104,7 +104,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 8faecad1dbdb9..235a413df686f 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -68,7 +68,7 @@ Default: `5`. `tokens_weight_threshold`:: (Optional, float) preview:[] -Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/query-dsl/weighted-tokens-query.asciidoc b/docs/reference/query-dsl/weighted-tokens-query.asciidoc index d4318665a9778..fb051f4229df6 100644 --- a/docs/reference/query-dsl/weighted-tokens-query.asciidoc +++ b/docs/reference/query-dsl/weighted-tokens-query.asciidoc @@ -58,7 +58,7 @@ This value must between 1 and 100. Default: `5`. `tokens_weight_threshold`:: -(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered nonsignificant and pruned. +(Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`. diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index c19f4f7cf989b..1496d7846a080 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -35,6 +35,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere ** These indices have many conflicting field mappings ** Many of those fields are included in the request These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. + To work around this issue, you have a number of options: ** Downgrade to an earlier version diff --git a/docs/reference/release-notes/8.15.1.asciidoc b/docs/reference/release-notes/8.15.1.asciidoc index 2c126cccbda9e..e3bfaa18b6986 100644 --- a/docs/reference/release-notes/8.15.1.asciidoc +++ b/docs/reference/release-notes/8.15.1.asciidoc @@ -15,6 +15,7 @@ can be configured using the https://www.elastic.co/guide/en/elasticsearch/refere ** These indices have many conflicting field mappings ** Many of those fields are included in the request These issues deplete heap memory, increasing the likelihood of OOM errors. (issue: {es-issue}111964[#111964], {es-issue}111358[#111358]). +In Kibana, you might indirectly execute these queries when using Discover, or adding a Field Statistics panel to a dashboard. + To work around this issue, you have a number of options: ** Downgrade to an earlier version @@ -23,6 +24,14 @@ To work around this issue, you have a number of options: <> ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts. +* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field +but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds: +** If the affected index already contains documents with a dense vector field, force merge the index to a single segment. +** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field +and then force merge to a single segment. +** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields. + + [[bug-8.15.1]] [float] === Bug fixes diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index 709d17091164c..de9a35e0d29b8 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -36,7 +36,11 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> { "service": "elser", <2> "service_settings": { - "num_allocations": 1, + "adaptive_allocations": { <3> + "enabled": true, + "min_number_of_allocations": 3, + "max_number_of_allocations": 10 + }, "num_threads": 1 } } @@ -46,6 +50,8 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> be used and ELSER creates sparse vectors. The `inference_id` is `my-elser-endpoint`. <2> The `elser` service is used in this example. +<3> This setting enables and configures {ml-docs}/ml-nlp-elser.html#elser-adaptive-allocations[adaptive allocations]. +Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. [NOTE] ==== diff --git a/libs/entitlement-runtime/README.md b/libs/entitlement-runtime/README.md new file mode 100644 index 0000000000000..49cbc873c9de5 --- /dev/null +++ b/libs/entitlement-runtime/README.md @@ -0,0 +1,14 @@ +### Entitlement runtime + +This module implements mechanisms to grant and check permissions under the _entitlements_ system. + +The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal. +The `entitlement-agent` tool instruments sensitive class library methods with calls to this module, in order to enforce the controls. + +This module is responsible for: +- Defining which class library methods are sensitive +- Defining what permissions should be checked for each sensitive method +- Implementing the permission checks +- Offering a "grant" API to grant permissions + +It is not responsible for anything to do with bytecode instrumentation; that responsibility lies with `entitlement-agent`. diff --git a/libs/plugin-classloader/build.gradle b/libs/entitlement-runtime/build.gradle similarity index 58% rename from libs/plugin-classloader/build.gradle rename to libs/entitlement-runtime/build.gradle index f54bec211286a..a552dd7d5ba47 100644 --- a/libs/plugin-classloader/build.gradle +++ b/libs/entitlement-runtime/build.gradle @@ -6,13 +6,19 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ - -// This is only required because :server needs this at runtime. -// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work. +apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.publish' -tasks.named("test").configure { enabled = false } +dependencies { + compileOnly project(':libs:elasticsearch-core') + + testImplementation project(":test:framework") +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} -// test depend on ES core... -tasks.named('forbiddenApisMain').configure { enabled = false} -tasks.named("jarHell").configure { enabled = false } +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/entitlement-runtime/src/main/java/module-info.java b/libs/entitlement-runtime/src/main/java/module-info.java new file mode 100644 index 0000000000000..13849f0658d72 --- /dev/null +++ b/libs/entitlement-runtime/src/main/java/module-info.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module org.elasticsearch.entitlement.runtime { + requires org.elasticsearch.base; + + exports org.elasticsearch.entitlement.runtime.api to org.elasticsearch.entitlement.agent; +} diff --git a/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java b/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java new file mode 100644 index 0000000000000..c06e1e5b1f858 --- /dev/null +++ b/libs/entitlement-runtime/src/main/java/org/elasticsearch/entitlement/runtime/api/EntitlementChecks.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.api; + +public class EntitlementChecks { + static boolean isAgentBooted = false; + + public static void setAgentBooted() { + isAgentBooted = true; + } + + public static boolean isAgentBooted() { + return isAgentBooted; + } +} diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 32d9945aff6e2..e99f5be0a1e6b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1415,7 +1415,7 @@ public void testNoTimestampInDocument() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON); Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); + assertThat(e.getCause().getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); } public void testMultipleTimestampValuesInDocument() throws Exception { @@ -1431,7 +1431,7 @@ public void testMultipleTimestampValuesInDocument() throws Exception { IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create") .source("{\"@timestamp\": [\"2020-12-12\",\"2022-12-12\"]}", XContentType.JSON); Exception e = expectThrows(Exception.class, client().index(indexRequest)); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + assertThat(e.getCause().getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); } public void testMixedAutoCreate() throws Exception { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index af3204ed443ab..cb5578a282dc9 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -33,10 +33,12 @@ teardown: --- "Redirect ingest failure in data stream to failure store": - requires: - cluster_features: ["gte_v8.15.0"] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [allowed_warnings, contains] - + reason: "Failure store status was added in 8.16+" + test_runner_features: [capabilities, allowed_warnings, contains] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: id: "failing_pipeline" @@ -92,6 +94,8 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} + - match: { _index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/'} - do: indices.get_data_stream: @@ -144,9 +148,12 @@ teardown: --- "Redirect shard failure in data stream to failure store": - requires: - cluster_features: ["gte_v8.14.0"] - reason: "data stream failure stores only redirect shard failures in 8.14+" - test_runner_features: [allowed_warnings, contains] + reason: "Failure store status was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: allowed_warnings: @@ -176,6 +183,8 @@ teardown: body: '@timestamp': '2020-12-12' count: 'invalid value' + - match: { _index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000002/'} + - match: { failure_store: used} - do: indices.get_data_stream: @@ -222,14 +231,13 @@ teardown: --- "Ensure failure is redirected to correct failure store after a reroute processor": - - skip: - known_issues: - - cluster_feature: "gte_v8.15.0" - fixed_by: "gte_v8.16.0" - reason: "Failure store documents contained the original index name rather than the rerouted one before v8.16.0" - requires: - test_runner_features: [allowed_warnings] - + test_runner_features: [allowed_warnings, capabilities] + reason: "Failure store status was added in 8.16+" + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: id: "failing_pipeline" @@ -307,6 +315,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: search: @@ -422,9 +431,12 @@ teardown: --- "Failure redirects to original failure store during index change if final pipeline changes target": - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] + reason: "Failure store status was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: @@ -466,6 +478,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: indices.get_data_stream: @@ -514,9 +527,12 @@ teardown: --- "Failure redirects to correct failure store when index loop is detected": - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] + reason: "Failure store status was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: @@ -591,6 +607,8 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { _index: '/\.fs-destination-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { failure_store: used} - do: @@ -640,9 +658,12 @@ teardown: --- "Failure redirects to correct failure store when pipeline loop is detected": - requires: - cluster_features: [ "gte_v8.15.0" ] - reason: "data stream failure stores REST structure changed in 8.15+" - test_runner_features: [ allowed_warnings, contains ] + reason: "Failure store status was added in 8.16+" + test_runner_features: [ capabilities, allowed_warnings, contains ] + capabilities: + - method: POST + path: /{index}/_doc + capabilities: [ 'failure_store_status' ] - do: ingest.put_pipeline: @@ -701,6 +722,7 @@ teardown: body: '@timestamp': '2020-12-12' foo: bar + - match: { failure_store: used} - do: indices.get_data_stream: @@ -752,9 +774,7 @@ teardown: --- "Version conflicts are not redirected to failure store": - requires: - cluster_features: ["gte_v8.16.0"] - reason: "Redirecting version conflicts to the failure store is considered a bug fixed in 8.16" - test_runner_features: [allowed_warnings, contains] + test_runner_features: [ allowed_warnings] - do: allowed_warnings: @@ -788,3 +808,92 @@ teardown: - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { items.1.create.status: 409 } - match: { items.1.create.error.type: version_conflict_engine_exception} + - is_false: items.1.create.failure_store + +--- +"Test failure store status with bulk request": + - requires: + test_runner_features: [ allowed_warnings, capabilities ] + reason: "Failure store status was added in 8.16+" + capabilities: + - method: POST + path: /_bulk + capabilities: [ 'failure_store_status' ] + - method: PUT + path: /_bulk + capabilities: [ 'failure_store_status' ] + + - do: + allowed_warnings: + - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: + failure_store: true + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + - do: + allowed_warnings: + - "index template [no-fs] has index patterns [no-fs*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [no-fs] will take precedence during new index creation" + indices.put_index_template: + name: no-fs + body: + index_patterns: no-fs* + data_stream: + failure_store: false + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + + + - do: + bulk: + refresh: true + body: + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "quick", "a": "brown", "b": "fox" }' + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "lazy", "a": "dog" }' + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "count": "invalid" }' + - '{ "create": { "_index": "no-fs", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "count": "invalid" }' + - is_true: errors + # Successfully indexed to backing index + - match: { items.0.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.0.create.status: 201 } + - is_false: items.1.create.failure_store + + # Rejected but not eligible to go to failure store + - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.1.create.status: 409 } + - match: { items.1.create.error.type: version_conflict_engine_exception} + - is_false: items.1.create.failure_store + + # Successfully indexed to failure store + - match: { items.2.create._index: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { items.2.create.status: 201 } + - match: { items.2.create.failure_store: used } + + # Rejected, eligible to go to failure store, but failure store not enabled + - match: { items.3.create._index: '/\.ds-no-fs-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.3.create.status: 400 } + - match: { items.3.create.error.type: document_parsing_exception } + - match: { items.3.create.failure_store: not_enabled } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java index 3c9e684ef4279..021ce09e0ed8e 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpAggregator.java @@ -46,7 +46,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception assert msg instanceof HttpObject; if (msg instanceof HttpRequest request) { var preReq = HttpHeadersAuthenticatorUtils.asHttpPreRequest(request); - aggregating = decider.test(preReq) && IGNORE_TEST.test(preReq); + aggregating = (decider.test(preReq) && IGNORE_TEST.test(preReq)) || request.decoderResult().isFailure(); } if (aggregating || msg instanceof FullHttpRequest) { super.channelRead(ctx, msg); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java index b04da46a2d7d7..a1aa211814520 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java @@ -17,6 +17,7 @@ import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaders; import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.codec.http.cookie.Cookie; import io.netty.handler.codec.http.cookie.ServerCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; @@ -48,6 +49,7 @@ public class Netty4HttpRequest implements HttpRequest { private final Exception inboundException; private final boolean pooled; private final int sequence; + private final QueryStringDecoder queryStringDecoder; Netty4HttpRequest(int sequence, io.netty.handler.codec.http.HttpRequest request, Netty4HttpRequestBodyStream contentStream) { this( @@ -94,6 +96,7 @@ private Netty4HttpRequest( this.pooled = pooled; this.released = released; this.inboundException = inboundException; + this.queryStringDecoder = new QueryStringDecoder(request.uri()); } @Override @@ -106,6 +109,11 @@ public String uri() { return request.uri(); } + @Override + public String rawPath() { + return queryStringDecoder.rawPath(); + } + @Override public HttpBody body() { assert released.get() == false; diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index c6e7fa3517771..5ed3d81392951 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -375,9 +375,8 @@ protected HttpMessage createMessage(String[] initialLine) throws Exception { final HttpObjectAggregator aggregator = new Netty4HttpAggregator( handlingSettings.maxContentLength(), httpPreRequest -> enabled.get() == false - || (httpPreRequest.uri().contains("_bulk") == false - || httpPreRequest.uri().contains("_bulk_update") - || httpPreRequest.uri().contains("/_xpack/monitoring/_bulk")) + || ((httpPreRequest.rawPath().endsWith("/_bulk") == false) + || httpPreRequest.rawPath().startsWith("/_xpack/monitoring/_bulk")) ); aggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); ch.pipeline() diff --git a/muted-tests.yml b/muted-tests.yml index a47fbcb6c9c89..e01d977a41c60 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -215,9 +215,6 @@ tests: - class: org.elasticsearch.xpack.sql.qa.security.JdbcSqlSpecIT method: test {case-functions.testSelectInsertWithLcaseAndLengthWithOrderBy} issue: https://github.com/elastic/elasticsearch/issues/112642 -- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT - method: testHistogramAggregation - issue: https://github.com/elastic/elasticsearch/issues/113109 - class: org.elasticsearch.action.admin.cluster.node.stats.NodeStatsTests method: testChunking issue: https://github.com/elastic/elasticsearch/issues/113139 @@ -235,18 +232,6 @@ tests: - class: org.elasticsearch.xpack.inference.rest.ServerSentEventsRestActionListenerTests method: testErrorMidStream issue: https://github.com/elastic/elasticsearch/issues/113179 -- class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests - method: testDataGeneratorProducesValidMappingAndDocument - issue: https://github.com/elastic/elasticsearch/issues/112966 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with Italian locale} - issue: https://github.com/elastic/elasticsearch/issues/113198 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {date.DateFormatLocale SYNC} - issue: https://github.com/elastic/elasticsearch/issues/113199 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {date.DateFormatLocale ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/113200 - class: org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests method: testHasPrivilegesOtherThanIndex issue: https://github.com/elastic/elasticsearch/issues/113202 @@ -271,12 +256,41 @@ tests: - class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests method: "testFold {TestCase= #2}" issue: https://github.com/elastic/elasticsearch/issues/113225 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: test {p0=esql/70_locale/Date format with default locale} - issue: https://github.com/elastic/elasticsearch/issues/113226 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates} - issue: https://github.com/elastic/elasticsearch/issues/113227 +- class: org.elasticsearch.index.mapper.DoubleRangeFieldMapperTests + method: testSyntheticSourceKeepAll + issue: https://github.com/elastic/elasticsearch/issues/113234 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testGetMappings + issue: https://github.com/elastic/elasticsearch/issues/113260 +- class: org.elasticsearch.xpack.security.authz.SecurityScrollTests + method: testSearchAndClearScroll + issue: https://github.com/elastic/elasticsearch/issues/113285 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + method: test {date.EvalDateFormatString SYNC} + issue: https://github.com/elastic/elasticsearch/issues/113293 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {date.EvalDateFormat} + issue: https://github.com/elastic/elasticsearch/issues/113294 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {date.DocsDateFormat} + issue: https://github.com/elastic/elasticsearch/issues/113295 +- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT + method: test {stats.DocsStatsGroupByMultipleValues} + issue: https://github.com/elastic/elasticsearch/issues/113296 +- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT + issue: https://github.com/elastic/elasticsearch/issues/113298 +- class: org.elasticsearch.integration.KibanaUserRoleIntegTests + method: testGetIndex + issue: https://github.com/elastic/elasticsearch/issues/113311 +- class: org.elasticsearch.packaging.test.WindowsServiceTests + method: test81JavaOptsInJvmOptions + issue: https://github.com/elastic/elasticsearch/issues/113313 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=esql/50_index_patterns/disjoint_mappings} + issue: https://github.com/elastic/elasticsearch/issues/113315 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=wildcard/10_wildcard_basic/Query_string wildcard query} + issue: https://github.com/elastic/elasticsearch/issues/113316 # Examples: # diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java index a7fa63709d580..edb29a8f4c98e 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiTokenizerFactory.java @@ -33,6 +33,7 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory { private static final String NBEST_COST = "nbest_cost"; private static final String NBEST_EXAMPLES = "nbest_examples"; private static final String DISCARD_COMPOUND_TOKEN = "discard_compound_token"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final Mode mode; @@ -58,7 +59,15 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]" ); } - List ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false, true); + List ruleList = Analysis.getWordList( + env, + settings, + USER_DICT_PATH_OPTION, + USER_DICT_RULES_OPTION, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback + true + ); if (ruleList == null || ruleList.isEmpty()) { return null; } @@ -66,6 +75,7 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting for (String line : ruleList) { sb.append(line).append(System.lineSeparator()); } + try (Reader rulesReader = new StringReader(sb.toString())) { return UserDictionary.open(rulesReader); } catch (IOException e) { diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java index 1229b4f348911..f26213d86c5a9 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/plugin/analysis/kuromoji/KuromojiAnalysisTests.java @@ -445,7 +445,26 @@ public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception { ) .build(); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[制限スピード] in user dictionary at line [4]")); + } + + public void testKuromojiAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "kuromoji") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .putList( + "index.analysis.analyzer.my_analyzer.user_dictionary_rules", + "c++,c++,w,w", + "#comment", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞", + "制限スピード,制限スピード,セイゲンスピード,テスト名詞" + ) + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "制限スピード")) { + assertTokenStreamContents(stream, new String[] { "制限スピード" }); + } } public void testDiscardCompoundToken() throws Exception { diff --git a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java index 8bc53fa69c9a7..ed8458bc94043 100644 --- a/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java +++ b/plugins/analysis-nori/src/main/java/org/elasticsearch/plugin/analysis/nori/NoriTokenizerFactory.java @@ -31,6 +31,7 @@ public class NoriTokenizerFactory extends AbstractTokenizerFactory { private static final String USER_DICT_PATH_OPTION = "user_dictionary"; private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules"; + private static final String LENIENT = "lenient"; private final UserDictionary userDictionary; private final KoreanTokenizer.DecompoundMode decompoundMode; @@ -54,7 +55,8 @@ public static UserDictionary getUserDictionary(Environment env, Settings setting settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, - true, + LENIENT, + false, // typically don't want to remove comments as deduplication will provide better feedback isSupportDuplicateCheck(indexSettings) ); if (ruleList == null || ruleList.isEmpty()) { diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java index e1123f167da99..1709d02263eea 100644 --- a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java +++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java @@ -127,7 +127,7 @@ public void testNoriAnalyzerDuplicateUserDictRule() throws Exception { .build(); final IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings)); - assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [3]")); + assertThat(exc.getMessage(), containsString("[세종] in user dictionary at line [4]")); } public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOException { @@ -144,6 +144,20 @@ public void testNoriAnalyzerDuplicateUserDictRuleWithLegacyVersion() throws IOEx } } + public void testNoriAnalyzerDuplicateUserDictRuleDeduplication() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "nori") + .put("index.analysis.analyzer.my_analyzer.lenient", "true") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.NORI_DUPLICATES) + .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C쁠쁠", "세종", "세종", "세종시 세종 시") + .build(); + TestAnalysis analysis = createTestAnalysis(settings); + Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer"); + try (TokenStream stream = analyzer.tokenStream("", "세종시")) { + assertTokenStreamContents(stream, new String[] { "세종", "시" }); + } + } + public void testNoriTokenizer() throws Exception { Settings settings = Settings.builder() .put("index.analysis.tokenizer.my_tokenizer.type", "nori_tokenizer") diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index e817290d71a6a..42a0a3d6da055 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -154,29 +154,29 @@ public QueryBuilderBWCIT(@Name("cluster") FullClusterRestartUpgradeStatus upgrad ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}]} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}]} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 0).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 0).addClause( + new SpanTermQueryBuilder("text_field", "value2") ) ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}], "slop": 2} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}], "slop": 2} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 2).addClause( + new SpanTermQueryBuilder("text_field", "value2") ) ); addCandidate( """ - "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \ - { "span_term": { "keyword_field": "value2" }}], "slop": 2, "in_order": false} + "span_near": {"clauses": [{ "span_term": { "text_field": "value1" }}, \ + { "span_term": { "text_field": "value2" }}], "slop": 2, "in_order": false} """, - new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause( - new SpanTermQueryBuilder("keyword_field", "value2") + new SpanNearQueryBuilder(new SpanTermQueryBuilder("text_field", "value1"), 2).addClause( + new SpanTermQueryBuilder("text_field", "value2") ).inOrder(false) ); } @@ -204,11 +204,6 @@ public void testQueryBuilderBWC() throws Exception { mappingsAndSettings.field("type", "percolator"); mappingsAndSettings.endObject(); } - { - mappingsAndSettings.startObject("keyword_field"); - mappingsAndSettings.field("type", "keyword"); - mappingsAndSettings.endObject(); - } { mappingsAndSettings.startObject("text_field"); mappingsAndSettings.field("type", "text"); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java index 08026e0435f33..2b24e53874e51 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IncrementalBulkRestIT.java @@ -29,6 +29,12 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) public class IncrementalBulkRestIT extends HttpSmokeTestCase { + public void testBulkUriMatchingDoesNotMatchBulkCapabilitiesApi() throws IOException { + Request request = new Request("GET", "/_capabilities?method=GET&path=%2F_bulk&capabilities=failure_store_status&pretty"); + Response response = getRestClient().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + public void testBulkMissingBody() throws IOException { Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"); request.setJsonEntity(""); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java index c652a9992d8fb..322c055de140b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -102,22 +101,9 @@ private void runTest(String actionName, String endpoint) throws Exception { private void updateClusterState(Function transformationFn) { final TimeValue timeout = TimeValue.timeValueSeconds(10); - - final AckedRequest ackedRequest = new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return timeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return timeout; - } - }; - PlainActionFuture future = new PlainActionFuture<>(); internalCluster().getAnyMasterNodeInstance(ClusterService.class) - .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(ackedRequest, future) { + .submitUnbatchedStateUpdateTask("get_mappings_cancellation_test", new AckedClusterStateUpdateTask(timeout, timeout, future) { @Override public ClusterState execute(ClusterState currentState) throws Exception { return transformationFn.apply(currentState); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 8a27eb3efd219..b5a9146bc54a6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1269,8 +1269,8 @@ synthetic_source with copy_to and ignored values: refresh: true body: name: "B" - k: ["5", "6"] - long: ["7", "8"] + k: ["55", "66"] + long: ["77", "88"] - do: search: @@ -1289,10 +1289,9 @@ synthetic_source with copy_to and ignored values: - match: hits.hits.1._source: name: "B" - k: ["5", "6"] - long: ["7", "8"] - - match: { hits.hits.1.fields.copy: ["5", "6", "7", "8"] } - + k: ["55", "66"] + long: ["77", "88"] + - match: { hits.hits.1.fields.copy: ["55", "66", "77", "88"] } --- synthetic_source with copy_to field having values in source: @@ -1553,3 +1552,420 @@ synthetic_source with copy_to and invalid values for copy: - match: { error.type: "document_parsing_exception" } - contains: { error.reason: "Copy-to currently only works for value-type fields" } + +--- +synthetic_source with copy_to pointing inside object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + my_values: + properties: + k: + type: keyword + ignore_above: 1 + copy_to: c.copy + long: + type: long + copy_to: c.copy + c: + properties: + copy: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + my_values: + k: "hello" + long: 100 + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy" ] + + - match: + hits.hits.0._source: + name: "A" + my_values: + k: "hello" + long: 100 + - match: + hits.hits.0.fields: + c.copy: [ "100", "hello" ] + + - match: + hits.hits.1._source: + name: "B" + my_values: + k: ["55", "66"] + long: [77, 88] + - match: + hits.hits.1.fields: + c.copy: ["55", "66", "77", "88"] + + - match: + hits.hits.2._source: + name: "C" + my_values: + k: "hello" + long: 100 + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy: [ "100", "hello", "zap" ] + +--- +synthetic_source with copy_to pointing to ambiguous field: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects false: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: false + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing to ambiguous field and subobjects auto: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + subobjects: auto + properties: + k: + type: keyword + copy_to: a.b.c + a: + properties: + b: + properties: + c: + type: keyword + b.c: + type: keyword + + - do: + index: + index: test + id: 1 + refresh: true + body: + k: "hey" + + - do: + search: + index: test + body: + docvalue_fields: [ "a.b.c" ] + + - match: + hits.hits.0._source: + k: "hey" + - match: + hits.hits.0.fields: + a.b.c: [ "hey" ] + +--- +synthetic_source with copy_to pointing at dynamic field: + - requires: + test_runner_features: contains + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + c: + properties: + f: + type: float + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + +--- +synthetic_source with copy_to pointing inside dynamic object: + - requires: + cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + k: + type: keyword + copy_to: c.copy + + - do: + index: + index: test + id: 1 + refresh: true + body: + name: "A" + k: "hello" + + - do: + index: + index: test + id: 2 + refresh: true + body: + name: "B" + k: ["55", "66"] + + - do: + index: + index: test + id: 3 + refresh: true + body: + name: "C" + k: "hello" + c: + copy: "zap" + + - do: + search: + index: test + sort: name + body: + docvalue_fields: [ "c.copy.keyword" ] + + - match: + hits.hits.0._source: + name: "A" + k: "hello" + - match: + hits.hits.0.fields: + c.copy.keyword: [ "hello" ] + + - match: + hits.hits.1._source: + name: "B" + k: ["55", "66"] + - match: + hits.hits.1.fields: + c.copy.keyword: [ "55", "66" ] + + - match: + hits.hits.2._source: + name: "C" + k: "hello" + c: + copy: "zap" + - match: + hits.hits.2.fields: + c.copy.keyword: [ "hello", "zap" ] + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index 7c345b7d4d3ac..975113953c995 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -1,6 +1,10 @@ --- "Test Index and Search locale dependent mappings / dates": + - requires: + test_runner_features: ["allowed_warnings"] - do: + allowed_warnings: + - "Date format [E, d MMM yyyy HH:mm:ss Z] contains textual field specifiers that could change in JDK 23" indices.create: index: test_index body: diff --git a/server/build.gradle b/server/build.gradle index 5492ca00e2d3b..5c12d47da8102 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -40,8 +40,6 @@ dependencies { api project(":libs:elasticsearch-tdigest") implementation project(":libs:elasticsearch-simdvec") - implementation project(':libs:elasticsearch-plugin-classloader') - // lucene api "org.apache.lucene:lucene-core:${versions.lucene}" api "org.apache.lucene:lucene-analysis-common:${versions.lucene}" diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index ac1c44de6dcf6..2a275cf563d86 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -255,9 +255,9 @@ private Set getShardIds(final String nodeId, final String indexName) { /** * Index documents until all the shards are at least WATERMARK_BYTES in size, and return the one with the smallest size */ - private ShardSizes createReasonableSizedShards(final String indexName) throws InterruptedException { + private ShardSizes createReasonableSizedShards(final String indexName) { while (true) { - indexRandom(true, indexName, scaledRandomIntBetween(100, 10000)); + indexRandom(false, indexName, scaledRandomIntBetween(100, 10000)); forceMerge(); refresh(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 61cf49ff6ca4e..9f1a094c80623 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; @@ -86,6 +87,7 @@ import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.recovery.RecoveryStats; @@ -137,8 +139,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; @@ -146,7 +150,10 @@ import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; +import static org.elasticsearch.index.MergePolicyConfig.INDEX_MERGE_ENABLED; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; +import static org.elasticsearch.indices.IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING; +import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -158,6 +165,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -1957,6 +1965,77 @@ public void accept(long globalCheckpoint, Exception e) { recoveryCompleteListener.onResponse(null); } + public void testPostRecoveryMerge() throws Exception { + internalCluster().startMasterOnlyNode(); + final var dataNode = internalCluster().startDataOnlyNode(); + final var indexName = randomIdentifier(); + createIndex(indexName, indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false).build()); + + final var initialSegmentCount = 20; + for (int i = 0; i < initialSegmentCount; i++) { + indexDoc(indexName, Integer.toString(i), "f", randomAlphaOfLength(10)); + refresh(indexName); // force a one-doc segment + } + flush(indexName); // commit all the one-doc segments + + final LongSupplier searchableSegmentCountSupplier = () -> indicesAdmin().prepareSegments(indexName) + .get(SAFE_AWAIT_TIMEOUT) + .getIndices() + .get(indexName) + .getShards() + .get(0) + .shards()[0].getSegments() + .stream() + .filter(Segment::isSearch) + .count(); + + assertEquals(initialSegmentCount, searchableSegmentCountSupplier.getAsLong()); + + // force a recovery by restarting the node, re-enabling merges while the node is down, but configure the node not to be in the hot + // or content tiers so that it does not do any post-recovery merge + internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + final var request = new UpdateSettingsRequest(Settings.builder().putNull(INDEX_MERGE_ENABLED).build(), indexName); + request.reopen(true); + safeGet(indicesAdmin().updateSettings(request)); + return Settings.builder() + .putList(NODE_ROLES_SETTING.getKey(), randomNonEmptySubsetOf(List.of("data_warm", "data_cold"))) + .build(); + } + }); + + ensureGreen(indexName); + final var mergeStats = indicesAdmin().prepareStats(indexName).clear().setMerge(true).get().getIndex(indexName).getShards()[0] + .getStats() + .getMerge(); + assertEquals(0, mergeStats.getCurrent()); + assertEquals(0, mergeStats.getTotal()); + assertEquals(initialSegmentCount, searchableSegmentCountSupplier.getAsLong()); + + // force a recovery by restarting the node again, but this time putting it into the hot or content tiers to enable post-recovery + // merges + internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + return Settings.builder() + .putList( + NODE_ROLES_SETTING.getKey(), + Stream.concat( + Stream.of(randomFrom("data", "data_content", "data_hot")), + Stream.of("data", "data_content", "data_hot", "data_warm", "data_cold").filter(p -> randomBoolean()) + ).distinct().toList() + ) + // set the inactive time to zero so that we flush immediately after every merge, rather than having the test wait 5min + .put(SHARD_INACTIVE_TIME_SETTING.getKey(), TimeValue.ZERO) + .build(); + } + }); + + ensureGreen(indexName); + assertBusy(() -> assertThat(searchableSegmentCountSupplier.getAsLong(), lessThan((long) initialSegmentCount))); + } + private void assertGlobalCheckpointIsStableAndSyncedInAllNodes(String indexName, List nodes, int shard) throws Exception { assertThat(nodes, is(not(empty()))); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 696624a4a8f27..4369c35c50ea3 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -25,7 +25,6 @@ requires org.elasticsearch.nativeaccess; requires org.elasticsearch.geo; requires org.elasticsearch.lz4; - requires org.elasticsearch.pluginclassloader; requires org.elasticsearch.securesm; requires org.elasticsearch.xcontent; requires org.elasticsearch.logging; diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index 95086c112d297..5d04e31069b1c 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.LockObtainFailedException; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper; @@ -1929,6 +1930,12 @@ private enum ElasticsearchExceptionHandle { org.elasticsearch.ingest.IngestPipelineException::new, 182, TransportVersions.INGEST_PIPELINE_EXCEPTION_ADDED + ), + INDEX_RESPONSE_WRAPPER_EXCEPTION( + IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, + IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus::new, + 183, + TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE ); final Class exceptionClass; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index a709f8b743343..2729ab856084d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -219,7 +219,8 @@ static TransportVersion def(int id) { public static final TransportVersion SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS = def(8_743_00_0); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_EMBEDDINGS_ADDED = def(8_744_00_0); public static final TransportVersion BULK_INCREMENTAL_STATE = def(8_745_00_0); - public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_746_00_0); + public static final TransportVersion FAILURE_STORE_STATUS_IN_INDEX_RESPONSE = def(8_746_00_0); + public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_747_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java index dc3ef791c7f3a..095ccd71fa266 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -9,6 +9,7 @@ package org.elasticsearch.action; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.support.WriteResponse; @@ -249,6 +250,10 @@ public String getLocation(@Nullable String routing) { return location.toString(); } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } + public void writeThin(StreamOutput out) throws IOException { super.writeTo(out); writeWithoutShardId(out); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index 71a989c8c9602..c0ceab139ff1b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -63,6 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); + failure.getFailureStoreStatus().toXContent(builder, params); builder.startObject(ERROR); ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause()); builder.endObject(); @@ -88,6 +89,7 @@ public static class Failure implements Writeable, ToXContentFragment { private final long seqNo; private final long term; private final boolean aborted; + private IndexDocFailureStoreStatus failureStoreStatus; /** * For write failures before operation was assigned a sequence number. @@ -103,7 +105,27 @@ public Failure(String index, String id, Exception cause) { ExceptionsHelper.status(cause), SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - false + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); + } + + /** + * For write failures before operation was assigned a sequence number. + * + * use @{link {@link #Failure(String, String, Exception, long, long)}} + * to record operation sequence no with failure + */ + public Failure(String index, String id, Exception cause, IndexDocFailureStoreStatus failureStoreStatus) { + this( + index, + id, + cause, + ExceptionsHelper.status(cause), + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + false, + failureStoreStatus ); } @@ -115,20 +137,48 @@ public Failure(String index, String id, Exception cause, boolean aborted) { ExceptionsHelper.status(cause), SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - aborted + aborted, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN ); } public Failure(String index, String id, Exception cause, RestStatus status) { - this(index, id, cause, status, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, false); + this( + index, + id, + cause, + status, + SequenceNumbers.UNASSIGNED_SEQ_NO, + SequenceNumbers.UNASSIGNED_PRIMARY_TERM, + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); } /** For write failures after operation was assigned a sequence number. */ public Failure(String index, String id, Exception cause, long seqNo, long term) { - this(index, id, cause, ExceptionsHelper.status(cause), seqNo, term, false); + this( + index, + id, + cause, + ExceptionsHelper.status(cause), + seqNo, + term, + false, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); } - private Failure(String index, String id, Exception cause, RestStatus status, long seqNo, long term, boolean aborted) { + private Failure( + String index, + String id, + Exception cause, + RestStatus status, + long seqNo, + long term, + boolean aborted, + IndexDocFailureStoreStatus failureStoreStatus + ) { this.index = index; this.id = id; this.cause = cause; @@ -136,6 +186,7 @@ private Failure(String index, String id, Exception cause, RestStatus status, lon this.seqNo = seqNo; this.term = term; this.aborted = aborted; + this.failureStoreStatus = failureStoreStatus; } /** @@ -154,6 +205,11 @@ public Failure(StreamInput in) throws IOException { seqNo = in.readZLong(); term = in.readVLong(); aborted = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } @Override @@ -167,6 +223,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeZLong(seqNo); out.writeVLong(term); out.writeBoolean(aborted); + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus.writeTo(out); + } } /** @@ -231,6 +290,14 @@ public boolean isAborted() { return aborted; } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return failureStoreStatus; + } + + public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus) { + this.failureStoreStatus = failureStoreStatus; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(INDEX_FIELD, index); @@ -244,6 +311,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws ElasticsearchException.generateThrowableXContent(builder, params, cause); builder.endObject(); builder.field(STATUS_FIELD, status.getStatus()); + failureStoreStatus.toXContent(builder, params); return builder; } @@ -341,6 +409,14 @@ public long getVersion() { return response.getVersion(); } + public IndexDocFailureStoreStatus getFailureStoreStatus() { + if (response != null) { + return response.getFailureStoreStatus(); + } else { + return failure.getFailureStoreStatus(); + } + } + /** * The actual response ({@link IndexResponse} or {@link DeleteResponse}). {@code null} in * case of failure. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 13229fbf65fef..1789acc1cb7a6 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.client.internal.OriginSettingClient; @@ -240,7 +241,13 @@ public void onFailure(Exception e) { if (failureStoreRedirect.index().equals(dataStream) == false) { continue; } - addFailure(failureStoreRedirect.request(), failureStoreRedirect.id(), failureStoreRedirect.index(), e); + addFailure( + failureStoreRedirect.request(), + failureStoreRedirect.id(), + failureStoreRedirect.index(), + e, + IndexDocFailureStoreStatus.FAILED + ); failedRolloverRequests.add(failureStoreRedirect.id()); } } @@ -323,7 +330,10 @@ private Map> groupRequestsByShards( shardRequests.add(bulkItemRequest); } catch (ElasticsearchParseException | IllegalArgumentException | RoutingMissingException | ResourceNotFoundException e) { String name = ia != null ? ia.getName() : docWriteRequest.index(); - addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, e); + var failureStoreStatus = isFailureStoreRequest(docWriteRequest) + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + addFailureAndDiscardRequest(docWriteRequest, bulkItemRequest.id(), name, e, failureStoreStatus); } } return requestsByShard; @@ -430,7 +440,10 @@ private void discardRedirectsAndFinish(Exception exception) { BulkItemResponse originalFailure = responses.get(slot); if (originalFailure.isFailed()) { originalFailure.getFailure().getCause().addSuppressed(exception); + originalFailure.getFailure().setFailureStoreStatus(IndexDocFailureStoreStatus.FAILED); } + // Always replace the item in the responses for thread visibility of any mutations + responses.set(slot, originalFailure); } completeBulkOperation(); } @@ -464,10 +477,19 @@ public void onResponse(BulkShardResponse bulkShardResponse) { if (bulkItemResponse.isFailed()) { assert bulkItemRequest.id() == bulkItemResponse.getItemId() : "Bulk items were returned out of order"; - processFailure(bulkItemRequest, getClusterState(), bulkItemResponse.getFailure().getCause()); + IndexDocFailureStoreStatus failureStoreStatus = processFailure( + bulkItemRequest, + getClusterState(), + bulkItemResponse.getFailure().getCause() + ); + bulkItemResponse.getFailure().setFailureStoreStatus(failureStoreStatus); addFailure(bulkItemResponse); } else { bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); + if (isFailureStoreRequest(bulkItemRequest.request()) + && bulkItemResponse.getResponse() instanceof IndexResponse ir) { + ir.setFailureStoreStatus(IndexDocFailureStoreStatus.USED); + } responses.set(bulkItemResponse.getItemId(), bulkItemResponse); } } @@ -498,13 +520,20 @@ private void handleShardFailure(BulkShardRequest bulkShardRequest, ClusterState for (BulkItemRequest request : bulkShardRequest.items()) { final String indexName = request.index(); DocWriteRequest docWriteRequest = request.request(); - - processFailure(request, clusterState, e); - addFailure(docWriteRequest, request.id(), indexName, e); + IndexDocFailureStoreStatus failureStoreStatus = processFailure(request, clusterState, e); + addFailure(docWriteRequest, request.id(), indexName, e, failureStoreStatus); } } - private void processFailure(BulkItemRequest bulkItemRequest, ClusterState clusterState, Exception cause) { + /** + * This method checks the eligibility of the failed document to be redirected to the failure store and updates the metrics. + * If the document is eligible it will call {@link BulkOperation#addDocumentToRedirectRequests}. Otherwise, it will return + * the appropriate failure store status and count this document as rejected. + * @return the status: + * - NOT_ENABLED, if the data stream didn't have the data store enabled and + * - FAILED if something went wrong in the preparation of the failure store request. + */ + private IndexDocFailureStoreStatus processFailure(BulkItemRequest bulkItemRequest, ClusterState clusterState, Exception cause) { var error = ExceptionsHelper.unwrapCause(cause); var errorType = ElasticsearchException.getExceptionName(error); DocWriteRequest docWriteRequest = bulkItemRequest.request(); @@ -513,25 +542,45 @@ private void processFailure(BulkItemRequest bulkItemRequest, ClusterState cluste // it has the failure store enabled. if (failureStoreCandidate != null) { // Do not redirect documents to a failure store that were already headed to one. - var isFailureStoreDoc = docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore(); - if (isFailureStoreDoc == false + var isFailureStoreRequest = isFailureStoreRequest(docWriteRequest); + if (isFailureStoreRequest == false && failureStoreCandidate.isFailureStoreEnabled() && error instanceof VersionConflictEngineException == false) { - // Redirect to failure store. + // Prepare the data stream failure store if necessary maybeMarkFailureStoreForRollover(failureStoreCandidate); - addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName()); - failureStoreMetrics.incrementFailureStore(bulkItemRequest.index(), errorType, FailureStoreMetrics.ErrorLocation.SHARD); + + // Enqueue the redirect to failure store. + boolean added = addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName()); + if (added) { + failureStoreMetrics.incrementFailureStore(bulkItemRequest.index(), errorType, FailureStoreMetrics.ErrorLocation.SHARD); + } else { + failureStoreMetrics.incrementRejected( + bulkItemRequest.index(), + errorType, + FailureStoreMetrics.ErrorLocation.SHARD, + isFailureStoreRequest + ); + return IndexDocFailureStoreStatus.FAILED; + } } else { // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled - // or this request was already targeting a failure store), we increment the rejected counter. + // or this request was already targeting a failure store), or this was a version conflict we increment the + // rejected counter. failureStoreMetrics.incrementRejected( bulkItemRequest.index(), errorType, FailureStoreMetrics.ErrorLocation.SHARD, - isFailureStoreDoc + isFailureStoreRequest ); + if (isFailureStoreRequest) { + return IndexDocFailureStoreStatus.FAILED; + } + if (failureStoreCandidate.isFailureStoreEnabled() == false) { + return IndexDocFailureStoreStatus.NOT_ENABLED; + } } } + return IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; } /** @@ -559,8 +608,9 @@ private static DataStream getRedirectTargetCandidate(DocWriteRequest docWrite * @param request The bulk item request that failed * @param cause The exception for the experienced the failure * @param failureStoreReference The data stream that contains the failure store for this item + * @return true, if adding the request to the queue was successful, false otherwise. */ - private void addDocumentToRedirectRequests(BulkItemRequest request, Exception cause, String failureStoreReference) { + private boolean addDocumentToRedirectRequests(BulkItemRequest request, Exception cause, String failureStoreReference) { // Convert the document into a failure document IndexRequest failureStoreRequest; try { @@ -585,12 +635,12 @@ private void addDocumentToRedirectRequests(BulkItemRequest request, Exception ca ); // Suppress and do not redirect cause.addSuppressed(ioException); - return; + return false; } // Store for second phase BulkItemRequest redirected = new BulkItemRequest(request.id(), failureStoreRequest); - failureStoreRedirects.add(redirected); + return failureStoreRedirects.add(redirected); } /** @@ -679,7 +729,7 @@ private boolean addFailureIfRequiresAliasAndAliasIsMissing(DocWriteRequest re "[" + DocWriteRequest.REQUIRE_ALIAS + "] request flag is [true] and [" + request.index() + "] is not an alias", request.index() ); - addFailureAndDiscardRequest(request, idx, request.index(), exception); + addFailureAndDiscardRequest(request, idx, request.index(), exception, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); return true; } return false; @@ -691,7 +741,7 @@ private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequ "[" + DocWriteRequest.REQUIRE_DATA_STREAM + "] request flag is [true] and [" + request.index() + "] is not a data stream", request.index() ); - addFailureAndDiscardRequest(request, idx, request.index(), exception); + addFailureAndDiscardRequest(request, idx, request.index(), exception, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); return true; } return false; @@ -700,7 +750,10 @@ private boolean addFailureIfRequiresDataStreamAndNoParentDataStream(DocWriteRequ private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index concreteIndex, int idx, final Metadata metadata) { IndexMetadata indexMetadata = metadata.getIndexSafe(concreteIndex); if (indexMetadata.getState() == IndexMetadata.State.CLOSE) { - addFailureAndDiscardRequest(request, idx, request.index(), new IndexClosedException(concreteIndex)); + var failureStoreStatus = isFailureStoreRequest(request) + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + addFailureAndDiscardRequest(request, idx, request.index(), new IndexClosedException(concreteIndex), failureStoreStatus); return true; } return false; @@ -709,18 +762,31 @@ private boolean addFailureIfIndexIsClosed(DocWriteRequest request, Index conc private boolean addFailureIfIndexCannotBeCreated(DocWriteRequest request, int idx) { IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); if (cannotCreate != null) { - addFailureAndDiscardRequest(request, idx, request.index(), cannotCreate); + var failureStoreStatus = isFailureStoreRequest(request) + ? IndexDocFailureStoreStatus.FAILED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + addFailureAndDiscardRequest(request, idx, request.index(), cannotCreate, failureStoreStatus); return true; } return false; } + private static boolean isFailureStoreRequest(DocWriteRequest request) { + return request instanceof IndexRequest ir && ir.isWriteToFailureStore(); + } + /** - * Like {@link BulkOperation#addFailure(DocWriteRequest, int, String, Exception)} but this method will remove the corresponding entry - * from the working bulk request so that it never gets processed again during this operation. + * Like {@link BulkOperation#addFailure(DocWriteRequest, int, String, Exception, IndexDocFailureStoreStatus)} but this method will + * remove the corresponding entry from the working bulk request so that it never gets processed again during this operation. */ - private void addFailureAndDiscardRequest(DocWriteRequest request, int idx, String index, Exception exception) { - addFailure(request, idx, index, exception); + private void addFailureAndDiscardRequest( + DocWriteRequest request, + int idx, + String index, + Exception exception, + IndexDocFailureStoreStatus failureStoreStatus + ) { + addFailure(request, idx, index, exception, failureStoreStatus); // make sure the request gets never processed again bulkRequest.requests.set(idx, null); } @@ -734,18 +800,26 @@ private void addFailureAndDiscardRequest(DocWriteRequest request, int idx, St * @param idx The slot of the bulk entry this request corresponds to * @param index The resource that this entry was being written to when it failed * @param exception The exception encountered for this entry + * @param failureStoreStatus The failure status as it was identified by this entry * @see BulkOperation#addFailure(BulkItemResponse) BulkOperation.addFailure if you have a bulk item response object already */ - private void addFailure(DocWriteRequest request, int idx, String index, Exception exception) { + private void addFailure( + DocWriteRequest request, + int idx, + String index, + Exception exception, + IndexDocFailureStoreStatus failureStoreStatus + ) { BulkItemResponse bulkItemResponse = responses.get(idx); if (bulkItemResponse == null) { - BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.id(), exception); + BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.id(), exception, failureStoreStatus); bulkItemResponse = BulkItemResponse.failure(idx, request.opType(), failure); } else { // Response already recorded. We should only be here if the existing response is a failure and // we are encountering a new failure while redirecting. assert bulkItemResponse.isFailed() : "Attempting to overwrite successful bulk item result with a failure"; bulkItemResponse.getFailure().getCause().addSuppressed(exception); + bulkItemResponse.getFailure().setFailureStoreStatus(failureStoreStatus); } // Always replace the item in the responses for thread visibility of any mutations responses.set(idx, bulkItemResponse); @@ -756,8 +830,8 @@ private void addFailure(DocWriteRequest request, int idx, String index, Excep * already, the failure information provided to this call will be added to the existing failure as a suppressed exception. * * @param bulkItemResponse the item response to add to the overall result array - * @see BulkOperation#addFailure(DocWriteRequest, int, String, Exception) BulkOperation.addFailure which conditionally creates the - * failure response only when one does not exist already + * @see BulkOperation#addFailure(DocWriteRequest, int, String, Exception, IndexDocFailureStoreStatus) BulkOperation.addFailure which + * conditionally creates the failure response only when one does not exist already */ private void addFailure(BulkItemResponse bulkItemResponse) { assert bulkItemResponse.isFailed() : "Attempting to add a successful bulk item response via the addFailure method"; @@ -767,6 +841,7 @@ private void addFailure(BulkItemResponse bulkItemResponse) { // we are encountering a new failure while redirecting. assert existingBulkItemResponse.isFailed() : "Attempting to overwrite successful bulk item result with a failure"; existingBulkItemResponse.getFailure().getCause().addSuppressed(bulkItemResponse.getFailure().getCause()); + existingBulkItemResponse.getFailure().setFailureStoreStatus(bulkItemResponse.getFailure().getFailureStoreStatus()); bulkItemResponse = existingBulkItemResponse; } // Always replace the item in the responses for thread visibility of any mutations diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java new file mode 100644 index 0000000000000..cb83d693a415b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/IndexDocFailureStoreStatus.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Locale; + +/** + * Captures the role of the failure store in this document response. For example, + * - USED, means that this document was stored in the failure store + * - NOT_ENABLED, means that this document was rejected by elasticsearch, but it could have been stored in + * the failure store has it been enabled. + * - FAILED, means that this failed document was eligible to be stored in the failure store and the failure store + * was enabled but something went wrong. + */ +public enum IndexDocFailureStoreStatus implements ToXContentFragment, Writeable { + /** + * This status represents that we have no information about this response or that the failure store is not applicable. + * For example: + * - when the doc was successfully indexed in a backing index of a data stream, + * - when we are running in a mixed version cluster and the information is not available, + * - when the doc was rejected by elasticsearch but failure store was not applicable (i.e. the target was an index). + */ + NOT_APPLICABLE_OR_UNKNOWN(0), + /** + * This status represents that this document was stored in the failure store successfully. + */ + USED(1), + /** + * This status represents that this document was rejected, but it could have ended up in the failure store if it was enabled. + */ + NOT_ENABLED(2), + /** + * This status represents that this document was rejected from the failure store. + */ + FAILED(3); + + private final byte id; + private final String label; + + IndexDocFailureStoreStatus(int id) { + this.id = (byte) id; + this.label = this.toString().toLowerCase(Locale.ROOT); + } + + public static IndexDocFailureStoreStatus read(StreamInput in) throws IOException { + return fromId(in.readByte()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte(id); + } + + /** + * @return id of the status, mainly used for wire serialisation purposes + */ + public byte getId() { + return id; + } + + /** + * @return the label of this status for display, just lowercase of the enum + */ + public String getLabel() { + return label; + } + + /** + * @param id a candidate id that (hopefully) can be converted to a FailureStoreStatus, used in wire serialisation + * @return the failure store status that corresponds to the id. + * @throws IllegalArgumentException when the id cannot produce a failure store status + */ + public static IndexDocFailureStoreStatus fromId(byte id) { + return switch (id) { + case 0 -> NOT_APPLICABLE_OR_UNKNOWN; + case 1 -> USED; + case 2 -> NOT_ENABLED; + case 3 -> FAILED; + default -> throw new IllegalArgumentException("Unknown failure store status: [" + id + "]"); + }; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // We avoid adding the not_applicable status in the response to not increase the size of bulk responses. + if (DataStream.isFailureStoreFeatureFlagEnabled() && this.equals(NOT_APPLICABLE_OR_UNKNOWN) == false) { + builder.field("failure_store", label); + } + return builder; + } + + /** + * Exception wrapper class that adds the failure store status in the XContent response. + * Note: We are not using {@link ExceptionWithFailureStoreStatus} because then it unwraps it directly + * in the {@link ElasticsearchException} and we cannot add the field. + */ + public static class ExceptionWithFailureStoreStatus extends ElasticsearchException { + + private final IndexDocFailureStoreStatus failureStoreStatus; + + public ExceptionWithFailureStoreStatus(BulkItemResponse.Failure failure) { + super(failure.getCause()); + this.failureStoreStatus = failure.getFailureStoreStatus(); + } + + public ExceptionWithFailureStoreStatus(StreamInput in) throws IOException { + super(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus = IndexDocFailureStoreStatus.fromId(in.readByte()); + } else { + failureStoreStatus = NOT_APPLICABLE_OR_UNKNOWN; + } + } + + @Override + protected void writeTo(StreamOutput out, Writer nestedExceptionsWriter) throws IOException { + super.writeTo(out, nestedExceptionsWriter); + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + out.writeByte(failureStoreStatus.getId()); + } + } + + @Override + protected XContentBuilder toXContent(XContentBuilder builder, Params params, int nestedLevel) throws IOException { + generateThrowableXContent(builder, params, this.getCause(), nestedLevel); + failureStoreStatus.toXContent(builder, params); + return builder; + } + + @Override + public RestStatus status() { + return ExceptionsHelper.status(getCause()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 9a79644e7cffd..03768af029141 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -193,7 +193,11 @@ public static ActionListe final Response response = (Response) bulkItemResponse.getResponse(); l.onResponse(response); } else { - l.onFailure(bulkItemResponse.getFailure().getCause()); + if (IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN.equals(bulkItemResponse.getFailure().getFailureStoreStatus())) { + l.onFailure(bulkItemResponse.getFailure().getCause()); + } else { + l.onFailure(new IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus(bulkItemResponse.getFailure())); + } } }); } @@ -211,7 +215,6 @@ protected void doInternalExecute( assert bulkRequest.getComponentTemplateSubstitutions().isEmpty() : "Component template substitutions are not allowed in a non-simulated bulk"; trackIndexRequests(bulkRequest); - Map indicesToAutoCreate = new HashMap<>(); Set dataStreamsToBeRolledOver = new HashSet<>(); Set failureStoresToBeRolledOver = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index efd159e91a60f..b98f5d87ee232 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -881,6 +881,9 @@ public IndexRequest setRequireAlias(boolean requireAlias) { return this; } + /** + * Transient flag denoting that the local request should be routed to a failure store. Not persisted across the wire. + */ public boolean isWriteToFailureStore() { return writeToFailureStore; } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java index 09888ba03e368..8d1bdf227e24d 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -36,6 +37,7 @@ public class IndexResponse extends DocWriteResponse { */ @Nullable protected final List executedPipelines; + private IndexDocFailureStoreStatus failureStoreStatus; public IndexResponse(ShardId shardId, StreamInput in) throws IOException { super(shardId, in); @@ -44,6 +46,11 @@ public IndexResponse(ShardId shardId, StreamInput in) throws IOException { } else { executedPipelines = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } public IndexResponse(StreamInput in) throws IOException { @@ -53,10 +60,15 @@ public IndexResponse(StreamInput in) throws IOException { } else { executedPipelines = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus = IndexDocFailureStoreStatus.read(in); + } else { + failureStoreStatus = IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; + } } public IndexResponse(ShardId shardId, String id, long seqNo, long primaryTerm, long version, boolean created) { - this(shardId, id, seqNo, primaryTerm, version, created, null); + this(shardId, id, seqNo, primaryTerm, version, created, null, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN); } public IndexResponse( @@ -68,7 +80,29 @@ public IndexResponse( boolean created, @Nullable List executedPipelines ) { - this(shardId, id, seqNo, primaryTerm, version, created ? Result.CREATED : Result.UPDATED, executedPipelines); + this( + shardId, + id, + seqNo, + primaryTerm, + version, + created ? Result.CREATED : Result.UPDATED, + executedPipelines, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); + } + + public IndexResponse( + ShardId shardId, + String id, + long seqNo, + long primaryTerm, + long version, + boolean created, + @Nullable List executedPipelines, + IndexDocFailureStoreStatus failureStoreStatus + ) { + this(shardId, id, seqNo, primaryTerm, version, created ? Result.CREATED : Result.UPDATED, executedPipelines, failureStoreStatus); } private IndexResponse( @@ -78,10 +112,12 @@ private IndexResponse( long primaryTerm, long version, Result result, - @Nullable List executedPipelines + @Nullable List executedPipelines, + IndexDocFailureStoreStatus failureStoreStatus ) { super(shardId, id, seqNo, primaryTerm, version, assertCreatedOrUpdated(result)); this.executedPipelines = executedPipelines; + this.failureStoreStatus = failureStoreStatus; } @Override @@ -90,6 +126,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus.writeTo(out); + } } @Override @@ -98,6 +137,9 @@ public void writeThin(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalCollection(executedPipelines, StreamOutput::writeString); } + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_STATUS_IN_INDEX_RESPONSE)) { + failureStoreStatus.writeTo(out); + } } public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { @@ -105,6 +147,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (executedPipelines != null) { updatedBuilder = updatedBuilder.field("executed_pipelines", executedPipelines.toArray()); } + failureStoreStatus.toXContent(builder, params); return updatedBuilder; } @@ -118,6 +161,15 @@ public RestStatus status() { return result == Result.CREATED ? RestStatus.CREATED : super.status(); } + public void setFailureStoreStatus(IndexDocFailureStoreStatus failureStoreStatus) { + this.failureStoreStatus = failureStoreStatus; + } + + @Override + public IndexDocFailureStoreStatus getFailureStoreStatus() { + return failureStoreStatus; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -129,6 +181,7 @@ public String toString() { builder.append(",seqNo=").append(getSeqNo()); builder.append(",primaryTerm=").append(getPrimaryTerm()); builder.append(",shards=").append(Strings.toString(getShardInfo())); + builder.append(",failure_store=").append(failureStoreStatus.getLabel()); return builder.append("]").toString(); } @@ -140,7 +193,16 @@ public String toString() { public static class Builder extends DocWriteResponse.Builder { @Override public IndexResponse build() { - IndexResponse indexResponse = new IndexResponse(shardId, id, seqNo, primaryTerm, version, result, null); + IndexResponse indexResponse = new IndexResponse( + shardId, + id, + seqNo, + primaryTerm, + version, + result, + null, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); indexResponse.setForcedRefresh(forcedRefresh); if (shardInfo != null) { indexResponse.setShardInfo(shardInfo); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java index 71359d569f5c6..1b930d5e8db3f 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -59,7 +60,16 @@ public SimulateIndexResponse( @Nullable Exception exception ) { // We don't actually care about most of the IndexResponse fields: - super(new ShardId(index, "", 0), id == null ? "" : id, 0, 0, version, true, pipelines); + super( + new ShardId(index, "", 0), + id == null ? "" : id, + 0, + 0, + version, + true, + pipelines, + IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN + ); this.source = source; this.sourceXContentType = sourceXContentType; setShardInfo(ShardInfo.EMPTY); diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java index dcb28b28f2b76..772f36898202b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.Nullable; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.dfs.AggregatedDfs; @@ -39,13 +40,15 @@ final class FetchSearchPhase extends SearchPhase { private final Logger logger; private final SearchProgressListener progressListener; private final AggregatedDfs aggregatedDfs; + @Nullable + private final SearchPhaseResults resultConsumer; private final SearchPhaseController.ReducedQueryPhase reducedQueryPhase; FetchSearchPhase( SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase ) { this( resultConsumer, @@ -64,7 +67,7 @@ final class FetchSearchPhase extends SearchPhase { SearchPhaseResults resultConsumer, AggregatedDfs aggregatedDfs, SearchPhaseContext context, - SearchPhaseController.ReducedQueryPhase reducedQueryPhase, + @Nullable SearchPhaseController.ReducedQueryPhase reducedQueryPhase, BiFunction, SearchPhase> nextPhaseFactory ) { super("fetch"); @@ -85,6 +88,7 @@ final class FetchSearchPhase extends SearchPhase { this.logger = context.getLogger(); this.progressListener = context.getTask().getProgressListener(); this.reducedQueryPhase = reducedQueryPhase; + this.resultConsumer = reducedQueryPhase == null ? resultConsumer : null; } @Override @@ -92,7 +96,7 @@ public void run() { context.execute(new AbstractRunnable() { @Override - protected void doRun() { + protected void doRun() throws Exception { innerRun(); } @@ -103,7 +107,10 @@ public void onFailure(Exception e) { }); } - private void innerRun() { + private void innerRun() throws Exception { + assert this.reducedQueryPhase == null ^ this.resultConsumer == null; + // depending on whether we executed the RankFeaturePhase we may or may not have the reduced query result computed already + final var reducedQueryPhase = this.reducedQueryPhase == null ? resultConsumer.reduce() : this.reducedQueryPhase; final int numShards = context.getNumShards(); // Usually when there is a single shard, we force the search type QUERY_THEN_FETCH. But when there's kNN, we might // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase. @@ -113,7 +120,7 @@ private void innerRun() { if (queryAndFetchOptimization) { assert assertConsistentWithQueryAndFetchOptimization(); // query AND fetch optimization - moveToNextPhase(searchPhaseShardResults); + moveToNextPhase(searchPhaseShardResults, reducedQueryPhase); } else { ScoreDoc[] scoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // no docs to fetch -- sidestep everything and return @@ -121,7 +128,7 @@ private void innerRun() { // we have to release contexts here to free up resources searchPhaseShardResults.asList() .forEach(searchPhaseShardResult -> releaseIrrelevantSearchContext(searchPhaseShardResult, context)); - moveToNextPhase(fetchResults.getAtomicArray()); + moveToNextPhase(fetchResults.getAtomicArray(), reducedQueryPhase); } else { final boolean shouldExplainRank = shouldExplainRankScores(context.getRequest()); final List> rankDocsPerShard = false == shouldExplainRank @@ -134,7 +141,7 @@ private void innerRun() { final CountedCollector counter = new CountedCollector<>( fetchResults, docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not - () -> moveToNextPhase(fetchResults.getAtomicArray()), + () -> moveToNextPhase(fetchResults.getAtomicArray(), reducedQueryPhase), context ); for (int i = 0; i < docIdsToLoad.length; i++) { @@ -243,7 +250,10 @@ public void onFailure(Exception e) { ); } - private void moveToNextPhase(AtomicArray fetchResultsArr) { + private void moveToNextPhase( + AtomicArray fetchResultsArr, + SearchPhaseController.ReducedQueryPhase reducedQueryPhase + ) { var resp = SearchPhaseController.merge(context.getRequest().scroll() != null, reducedQueryPhase, fetchResultsArr); context.addReleasable(resp::decRef); fetchResults.close(); diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 49e5c1b6d69e3..81053a70eca9f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -70,6 +70,12 @@ public class RankFeaturePhase extends SearchPhase { @Override public void run() { + RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); + if (rankFeaturePhaseRankCoordinatorContext == null) { + moveToNextPhase(queryPhaseResults, null); + return; + } + context.execute(new AbstractRunnable() { @Override protected void doRun() throws Exception { @@ -77,7 +83,7 @@ protected void doRun() throws Exception { // was set up at FetchSearchPhase. // we do the heavy lifting in this inner run method where we reduce aggs etc - innerRun(); + innerRun(rankFeaturePhaseRankCoordinatorContext); } @Override @@ -87,51 +93,39 @@ public void onFailure(Exception e) { }); } - void innerRun() throws Exception { + void innerRun(RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext) throws Exception { // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call // to operate on the first `rank_window_size * num_shards` results and merge them appropriately. SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce(); - RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); - if (rankFeaturePhaseRankCoordinatorContext != null) { - ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size - final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); - final CountedCollector rankRequestCounter = new CountedCollector<>( - rankPhaseResults, - context.getNumShards(), - () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), - context - ); + ScoreDoc[] queryScoreDocs = reducedQueryPhase.sortedTopDocs().scoreDocs(); // rank_window_size + final List[] docIdsToLoad = SearchPhaseController.fillDocIdsToLoad(context.getNumShards(), queryScoreDocs); + final CountedCollector rankRequestCounter = new CountedCollector<>( + rankPhaseResults, + context.getNumShards(), + () -> onPhaseDone(rankFeaturePhaseRankCoordinatorContext, reducedQueryPhase), + context + ); - // we send out a request to each shard in order to fetch the needed feature info - for (int i = 0; i < docIdsToLoad.length; i++) { - List entry = docIdsToLoad[i]; - SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); - if (entry == null || entry.isEmpty()) { - if (queryResult != null) { - releaseIrrelevantSearchContext(queryResult, context); - progressListener.notifyRankFeatureResult(i); - } - rankRequestCounter.countDown(); - } else { - executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); + // we send out a request to each shard in order to fetch the needed feature info + for (int i = 0; i < docIdsToLoad.length; i++) { + List entry = docIdsToLoad[i]; + SearchPhaseResult queryResult = queryPhaseResults.getAtomicArray().get(i); + if (entry == null || entry.isEmpty()) { + if (queryResult != null) { + releaseIrrelevantSearchContext(queryResult, context); + progressListener.notifyRankFeatureResult(i); } + rankRequestCounter.countDown(); + } else { + executeRankFeatureShardPhase(queryResult, rankRequestCounter, entry); } - } else { - moveToNextPhase(queryPhaseResults, reducedQueryPhase); } } private RankFeaturePhaseRankCoordinatorContext coordinatorContext(SearchSourceBuilder source) { return source == null || source.rankBuilder() == null ? null - : context.getRequest() - .source() - .rankBuilder() - .buildRankFeaturePhaseCoordinatorContext( - context.getRequest().source().size(), - context.getRequest().source().from(), - client - ); + : source.rankBuilder().buildRankFeaturePhaseCoordinatorContext(source.size(), source.from(), client); } private void executeRankFeatureShardPhase( diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java index a69c4470c2add..11872cd85d9ba 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.support.master; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; @@ -23,9 +22,7 @@ * Abstract base class for action requests that track acknowledgements of cluster state updates: such a request is acknowledged only once * the cluster state update is committed and all relevant nodes have applied it and acknowledged its application to the elected master.. */ -public abstract class AcknowledgedRequest> extends MasterNodeRequest - implements - AckedRequest { +public abstract class AcknowledgedRequest> extends MasterNodeRequest { public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30); @@ -74,7 +71,6 @@ public final Request ackTimeout(TimeValue ackTimeout) { /** * @return the current ack timeout as a {@link TimeValue} */ - @Override public final TimeValue ackTimeout() { return ackTimeout; } diff --git a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java index 34d7b6e913dec..de42591c15d27 100644 --- a/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java @@ -9,8 +9,8 @@ package org.elasticsearch.cluster; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Priority; import org.elasticsearch.core.TimeValue; @@ -23,21 +23,38 @@ public abstract class AckedClusterStateUpdateTask extends ClusterStateUpdateTask implements ClusterStateAckListener { private final ActionListener listener; - private final AckedRequest request; + private final TimeValue ackTimeout; - protected AckedClusterStateUpdateTask(AckedRequest request, ActionListener listener) { - this(Priority.NORMAL, request, listener); + protected AckedClusterStateUpdateTask(AcknowledgedRequest request, ActionListener listener) { + this(Priority.NORMAL, request.masterNodeTimeout(), request.ackTimeout(), listener); + } + + protected AckedClusterStateUpdateTask( + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + ActionListener listener + ) { + this(Priority.NORMAL, masterNodeTimeout, ackTimeout, listener); + } + + protected AckedClusterStateUpdateTask( + Priority priority, + AcknowledgedRequest request, + ActionListener listener + ) { + this(priority, request.masterNodeTimeout(), request.ackTimeout(), listener); } @SuppressWarnings("unchecked") protected AckedClusterStateUpdateTask( Priority priority, - AckedRequest request, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, ActionListener listener ) { - super(priority, request.masterNodeTimeout()); + super(priority, masterNodeTimeout); this.listener = (ActionListener) listener; - this.request = request; + this.ackTimeout = ackTimeout; } /** @@ -81,6 +98,6 @@ public void onFailure(Exception e) { * Acknowledgement timeout, maximum time interval to wait for acknowledgements */ public final TimeValue ackTimeout() { - return request.ackTimeout(); + return ackTimeout; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java index 69a51c80839d0..8841b315b0138 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/cluster/ack/ClusterStateUpdateRequest.java @@ -15,7 +15,7 @@ * Base class to be used when needing to update the cluster state * Contains the basic fields that are always needed */ -public abstract class ClusterStateUpdateRequest> implements AckedRequest { +public abstract class ClusterStateUpdateRequest> { private TimeValue ackTimeout; private TimeValue masterNodeTimeout; @@ -23,7 +23,6 @@ public abstract class ClusterStateUpdateRequest(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-data-stream [" + request.name + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { // When we're manually creating a data stream (i.e. not an auto creation), we don't need to initialize the failure store @@ -138,13 +142,19 @@ public ClusterState createDataStream( ); } - public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest< - CreateDataStreamClusterStateUpdateRequest> { - - private final boolean performReroute; - private final String name; - private final long startTime; - private final SystemDataStreamDescriptor descriptor; + public record CreateDataStreamClusterStateUpdateRequest( + String name, + long startTime, + @Nullable SystemDataStreamDescriptor systemDataStreamDescriptor, + TimeValue masterNodeTimeout, + TimeValue ackTimeout, + boolean performReroute + ) { + public CreateDataStreamClusterStateUpdateRequest { + Objects.requireNonNull(name); + Objects.requireNonNull(masterNodeTimeout); + Objects.requireNonNull(ackTimeout); + } public CreateDataStreamClusterStateUpdateRequest(String name) { this(name, System.currentTimeMillis(), null, TimeValue.ZERO, TimeValue.ZERO, true); @@ -154,42 +164,14 @@ public CreateDataStreamClusterStateUpdateRequest( String name, SystemDataStreamDescriptor systemDataStreamDescriptor, TimeValue masterNodeTimeout, - TimeValue timeout, + TimeValue ackTimeout, boolean performReroute ) { - this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, timeout, performReroute); - } - - public CreateDataStreamClusterStateUpdateRequest( - String name, - long startTime, - SystemDataStreamDescriptor systemDataStreamDescriptor, - TimeValue masterNodeTimeout, - TimeValue timeout, - boolean performReroute - ) { - this.name = name; - this.startTime = startTime; - this.descriptor = systemDataStreamDescriptor; - this.performReroute = performReroute; - masterNodeTimeout(masterNodeTimeout); - ackTimeout(timeout); + this(name, System.currentTimeMillis(), systemDataStreamDescriptor, masterNodeTimeout, ackTimeout, performReroute); } public boolean isSystem() { - return descriptor != null; - } - - public boolean performReroute() { - return performReroute; - } - - public SystemDataStreamDescriptor getSystemDataStreamDescriptor() { - return descriptor; - } - - long getStartTime() { - return startTime; + return systemDataStreamDescriptor != null; } } @@ -238,7 +220,7 @@ static ClusterState createDataStream( boolean initializeFailureStore ) throws Exception { String dataStreamName = request.name; - SystemDataStreamDescriptor systemDataStreamDescriptor = request.getSystemDataStreamDescriptor(); + SystemDataStreamDescriptor systemDataStreamDescriptor = request.systemDataStreamDescriptor(); boolean isSystemDataStreamName = metadataCreateIndexService.getSystemIndices().isSystemDataStream(request.name); assert (isSystemDataStreamName && systemDataStreamDescriptor != null) || (isSystemDataStreamName == false && systemDataStreamDescriptor == null) @@ -287,13 +269,13 @@ static ClusterState createDataStream( if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } - String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.getStartTime()); + String failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, initialGeneration, request.startTime()); currentState = createFailureStoreIndex( metadataCreateIndexService, "initialize_data_stream", settings, currentState, - request.getStartTime(), + request.startTime(), dataStreamName, template, failureStoreIndexName, @@ -303,7 +285,7 @@ static ClusterState createDataStream( } if (writeIndex == null) { - String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.getStartTime()); + String firstBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, initialGeneration, request.startTime()); currentState = createBackingIndex( metadataCreateIndexService, currentState, @@ -392,7 +374,7 @@ private static ClusterState createBackingIndex( firstBackingIndexName ).dataStreamName(dataStreamName) .systemDataStreamDescriptor(systemDataStreamDescriptor) - .nameResolvedInstant(request.getStartTime()) + .nameResolvedInstant(request.startTime()) .performReroute(request.performReroute()) .setMatchingTemplate(template); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 275c186a1ea85..061aa18dd464a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -296,7 +296,12 @@ private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request, var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext()); submitUnbatchedTask( "create-index [" + request.index() + "], cause [" + request.cause() + "]", - new AckedClusterStateUpdateTask(Priority.URGENT, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.URGENT, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 1c93a13583e94..7ab68abb99ca5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -105,7 +105,12 @@ public void migrateToDataStream( var delegate = new AllocationActionListener<>(listener, threadContext); submitUnbatchedTask( "migrate-to-data-stream [" + request.aliasName + "]", - new AckedClusterStateUpdateTask(Priority.HIGH, request, delegate.clusterStateUpdate()) { + new AckedClusterStateUpdateTask( + Priority.HIGH, + request.masterNodeTimeout(), + request.ackTimeout(), + delegate.clusterStateUpdate() + ) { @Override public ClusterState execute(ClusterState currentState) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java index 893c6b9714eaa..0aacc466cce7a 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java @@ -177,6 +177,38 @@ public void writeWithSizePrefix(Writeable writeable) throws IOException { } } + // overridden with some code duplication the same way other write methods in this class are overridden to bypass StreamOutput's + // intermediary buffers + @Override + public void writeString(String str) throws IOException { + final int currentPageOffset = this.currentPageOffset; + final int charCount = str.length(); + // maximum serialized length is 3 bytes per char + 5 bytes for the longest possible vint + if (charCount * 3 + 5 > (pageSize - currentPageOffset)) { + super.writeString(str); + return; + } + BytesRef currentPage = pages.get(pageIndex).v(); + int off = currentPage.offset + currentPageOffset; + byte[] buffer = currentPage.bytes; + // mostly duplicated from StreamOutput.writeString to to get more reliable compilation of this very hot loop + int offset = off + putVInt(buffer, charCount, off); + for (int i = 0; i < charCount; i++) { + final int c = str.charAt(i); + if (c <= 0x007F) { + buffer[offset++] = ((byte) c); + } else if (c > 0x07FF) { + buffer[offset++] = ((byte) (0xE0 | c >> 12 & 0x0F)); + buffer[offset++] = ((byte) (0x80 | c >> 6 & 0x3F)); + buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F)); + } else { + buffer[offset++] = ((byte) (0xC0 | c >> 6 & 0x1F)); + buffer[offset++] = ((byte) (0x80 | c >> 0 & 0x3F)); + } + } + this.currentPageOffset = offset - currentPage.offset; + } + @Override public void flush() { // nothing to do diff --git a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java index 77454c5686538..ccf375aec60a5 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java +++ b/server/src/main/java/org/elasticsearch/http/HttpPreRequest.java @@ -33,6 +33,24 @@ public interface HttpPreRequest { */ String uri(); + /** + * The uri without the query string. + */ + default String rawPath() { + String uri = uri(); + final int index = uri.indexOf('?'); + if (index >= 0) { + return uri.substring(0, index); + } else { + final int index2 = uri.indexOf('#'); + if (index2 >= 0) { + return uri.substring(0, index2); + } else { + return uri; + } + } + } + /** * Get all of the headers and values associated with the HTTP headers. * Modifications of this map are not supported. diff --git a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java index 1a90f5f110376..462490a7fceb7 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.analysis; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.ar.ArabicAnalyzer; import org.apache.lucene.analysis.bg.BulgarianAnalyzer; @@ -67,6 +69,7 @@ import java.security.AccessControlException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -78,6 +81,7 @@ public class Analysis { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(Analysis.class); + private static final Logger logger = LogManager.getLogger(Analysis.class); public static void checkForDeprecatedVersion(String name, Settings settings) { String sVersion = settings.get("version"); @@ -267,12 +271,14 @@ public static List getWordList( Settings settings, String settingPath, String settingList, + String settingLenient, boolean removeComments, boolean checkDuplicate ) { + boolean deduplicateDictionary = settings.getAsBoolean(settingLenient, false); final List ruleList = getWordList(env, settings, settingPath, settingList, removeComments); if (ruleList != null && ruleList.isEmpty() == false && checkDuplicate) { - checkDuplicateRules(ruleList); + return deDuplicateRules(ruleList, deduplicateDictionary == false); } return ruleList; } @@ -288,24 +294,36 @@ public static List getWordList( * If the addition to the HashSet returns false, it means that item was already present in the set, indicating a duplicate. * In such a case, an IllegalArgumentException is thrown specifying the duplicate term and the line number in the original list. * + * Optionally the function will return the deduplicated list + * * @param ruleList The list of rules to check for duplicates. * @throws IllegalArgumentException If a duplicate rule is found. */ - private static void checkDuplicateRules(List ruleList) { - Set dup = new HashSet<>(); - int lineNum = 0; - for (String line : ruleList) { - // ignore comments + private static List deDuplicateRules(List ruleList, boolean failOnDuplicate) { + Set duplicateKeys = new HashSet<>(); + List deduplicatedList = new ArrayList<>(); + for (int lineNum = 0; lineNum < ruleList.size(); lineNum++) { + String line = ruleList.get(lineNum); + // ignore lines beginning with # as those are comments if (line.startsWith("#") == false) { String[] values = CSVUtil.parse(line); - if (dup.add(values[0]) == false) { - throw new IllegalArgumentException( - "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + lineNum + "]" - ); + if (duplicateKeys.add(values[0]) == false) { + if (failOnDuplicate) { + throw new IllegalArgumentException( + "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + (lineNum + 1) + "]" + ); + } else { + logger.warn("Ignoring duplicate term [" + values[0] + "] in user dictionary " + "at line [" + (lineNum + 1) + "]"); + } + } else { + deduplicatedList.add(line); } + } else { + deduplicatedList.add(line); } - ++lineNum; } + + return Collections.unmodifiableList(deduplicatedList); } private static List loadWordList(Path path, boolean removeComments) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java index 3aaf2ee5a8c4b..2f4af42041fce 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java @@ -265,9 +265,7 @@ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags) return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state); } - if (indexHasPositions - && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) - && (indexHasOffsets == false || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) + if ((indexHasOffsets == false || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) && (indexHasPayloads == false || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) { return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state); } @@ -287,8 +285,6 @@ final class BlockDocsEnum extends PostingsEnum { private ES812SkipReader skipper; private boolean skipped; - final IndexInput startDocIn; - IndexInput docIn; final boolean indexHasFreq; final boolean indexHasPos; @@ -320,8 +316,7 @@ final class BlockDocsEnum extends PostingsEnum { private boolean isFreqsRead; private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1 - BlockDocsEnum(FieldInfo fieldInfo) throws IOException { - this.startDocIn = ES812PostingsReader.this.docIn; + BlockDocsEnum(FieldInfo fieldInfo) { this.docIn = null; indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; indexHasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; @@ -333,7 +328,7 @@ final class BlockDocsEnum extends PostingsEnum { } public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { - return docIn == startDocIn + return docIn == ES812PostingsReader.this.docIn && indexHasFreq == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0) && indexHasPos == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) && indexHasPayloads == fieldInfo.hasPayloads(); @@ -348,7 +343,7 @@ public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOExcep if (docFreq > 1) { if (docIn == null) { // lazy init - docIn = startDocIn.clone(); + docIn = ES812PostingsReader.this.docIn.clone(); } docIn.seek(docTermStartFP); } @@ -379,22 +374,22 @@ public int freq() throws IOException { } @Override - public int nextPosition() throws IOException { + public int nextPosition() { return -1; } @Override - public int startOffset() throws IOException { + public int startOffset() { return -1; } @Override - public int endOffset() throws IOException { + public int endOffset() { return -1; } @Override - public BytesRef getPayload() throws IOException { + public BytesRef getPayload() { return null; } @@ -604,7 +599,7 @@ final class EverythingEnum extends PostingsEnum { private boolean needsPayloads; // true if we actually need payloads private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1 - EverythingEnum(FieldInfo fieldInfo) throws IOException { + EverythingEnum(FieldInfo fieldInfo) { indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; indexHasPayloads = fieldInfo.hasPayloads(); @@ -690,7 +685,7 @@ public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOExc } @Override - public int freq() throws IOException { + public int freq() { return freq; } @@ -851,16 +846,13 @@ public int advance(int target) throws IOException { // Now scan: long doc; - while (true) { + do { doc = docBuffer[docBufferUpto]; freq = (int) freqBuffer[docBufferUpto]; posPendingCount += freq; docBufferUpto++; - if (doc >= target) { - break; - } - } + } while (doc < target); position = 0; lastStartOffset = 0; @@ -1163,7 +1155,7 @@ public int advance(int target) throws IOException { } @Override - public int nextPosition() throws IOException { + public int nextPosition() { return -1; } @@ -1223,16 +1215,6 @@ final class BlockImpactsPostingsEnum extends ImpactsEnum { // before reading positions: private long posPendingFP; - // Where this term's postings start in the .doc file: - private final long docTermStartFP; - - // Where this term's postings start in the .pos file: - private final long posTermStartFP; - - // Where this term's payloads/offsets start in the .pay - // file: - private final long payTermStartFP; - // File pointer where the last (vInt encoded) pos delta // block is. We need this to know whether to bulk // decode vs vInt decode the block: @@ -1251,9 +1233,13 @@ final class BlockImpactsPostingsEnum extends ImpactsEnum { this.posIn = ES812PostingsReader.this.posIn.clone(); docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; + // Where this term's postings start in the .doc file: + long docTermStartFP = termState.docStartFP; + // Where this term's postings start in the .pos file: + long posTermStartFP = termState.posStartFP; + // Where this term's payloads/offsets start in the .pay + // file: + long payTermStartFP = termState.payStartFP; totalTermFreq = termState.totalTermFreq; docIn.seek(docTermStartFP); posPendingFP = posTermStartFP; @@ -1276,7 +1262,7 @@ final class BlockImpactsPostingsEnum extends ImpactsEnum { } @Override - public int freq() throws IOException { + public int freq() { return freq; } @@ -1523,16 +1509,6 @@ final class BlockImpactsEverythingEnum extends ImpactsEnum { // before reading payloads/offsets: private long payPendingFP; - // Where this term's postings start in the .doc file: - private final long docTermStartFP; - - // Where this term's postings start in the .pos file: - private final long posTermStartFP; - - // Where this term's payloads/offsets start in the .pay - // file: - private final long payTermStartFP; - // File pointer where the last (vInt encoded) pos delta // block is. We need this to know whether to bulk // decode vs vInt decode the block: @@ -1593,20 +1569,17 @@ final class BlockImpactsEverythingEnum extends ImpactsEnum { } docFreq = termState.docFreq; - docTermStartFP = termState.docStartFP; - posTermStartFP = termState.posStartFP; - payTermStartFP = termState.payStartFP; totalTermFreq = termState.totalTermFreq; - docIn.seek(docTermStartFP); - posPendingFP = posTermStartFP; - payPendingFP = payTermStartFP; + docIn.seek(termState.docStartFP); + posPendingFP = termState.posStartFP; + payPendingFP = termState.payStartFP; posPendingCount = 0; if (termState.totalTermFreq < BLOCK_SIZE) { - lastPosBlockFP = posTermStartFP; + lastPosBlockFP = termState.posStartFP; } else if (termState.totalTermFreq == BLOCK_SIZE) { lastPosBlockFP = -1; } else { - lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; + lastPosBlockFP = termState.posStartFP + termState.lastPosBlockOffset; } doc = -1; @@ -1617,7 +1590,13 @@ final class BlockImpactsEverythingEnum extends ImpactsEnum { docBufferUpto = BLOCK_SIZE; skipper = new ES812ScoreSkipReader(docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads); - skipper.init(docTermStartFP + termState.skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq); + skipper.init( + termState.docStartFP + termState.skipOffset, + termState.docStartFP, + termState.posStartFP, + termState.payStartFP, + docFreq + ); if (indexHasFreq == false) { for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 4d1b68214eddb..c2970d8716147 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -123,6 +123,7 @@ public int get() { private Field version; private final SeqNoFieldMapper.SequenceIDFields seqID; private final Set fieldsAppliedFromTemplates; + /** * Fields that are copied from values of other fields via copy_to. * This per-document state is needed since it is possible @@ -453,26 +454,16 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) { - /* - Mark this field as containing copied data meaning it should not be present - in synthetic _source (to be consistent with stored _source). - Ignored source values take precedence over standard synthetic source implementation - so by adding this nothing entry we "disable" field in synthetic source. - Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values - in copied field after reindexing. - - Note that this applies to fields that are copied from fields using ignored source themselves - and therefore we don't check for canAddIgnoredField(). - */ - ignoredFieldValues.add(IgnoredSourceFieldMapper.NameValue.fromContext(this, fieldName, XContentDataHelper.nothing())); - } } public boolean isCopyToDestinationField(String name) { return copyToFields.contains(name); } + public Set getCopyToFields() { + return copyToFields; + } + /** * Add a new mapper dynamically created while parsing. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 7b926b091b3a2..d57edb757ba10 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -26,9 +26,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -113,6 +112,10 @@ NameValue cloneWithValue(BytesRef value) { assert value() == null; return new NameValue(name, parentOffset, value, doc); } + + boolean hasValue() { + return XContentDataHelper.isDataPresent(value); + } } static final class IgnoredValuesFieldMapperType extends StringFieldType { @@ -147,11 +150,38 @@ protected String contentType() { @Override public void postParse(DocumentParserContext context) { // Ignored values are only expected in synthetic mode. - assert context.getIgnoredFieldValues().isEmpty() || context.mappingLookup().isSourceSynthetic(); - List ignoredFieldValues = new ArrayList<>(context.getIgnoredFieldValues()); - // ensure consistent ordering when retrieving synthetic source - Collections.sort(ignoredFieldValues, Comparator.comparing(NameValue::name)); - for (NameValue nameValue : ignoredFieldValues) { + if (context.mappingLookup().isSourceSynthetic() == false) { + assert context.getIgnoredFieldValues().isEmpty(); + return; + } + + Collection ignoredValuesToWrite = context.getIgnoredFieldValues(); + if (context.getCopyToFields().isEmpty() == false && indexSettings.getSkipIgnoredSourceWrite() == false) { + /* + Mark fields as containing copied data meaning they should not be present + in synthetic _source (to be consistent with stored _source). + Ignored source values take precedence over standard synthetic source implementation + so by adding the `XContentDataHelper.voidValue()` entry we disable the field in synthetic source. + Otherwise, it would be constructed f.e. from doc_values which leads to duplicate values + in copied field after reindexing. + */ + var mutableList = new ArrayList<>(ignoredValuesToWrite); + for (String copyToField : context.getCopyToFields()) { + ObjectMapper parent = context.parent().findParentMapper(copyToField); + if (parent == null) { + // There are scenarios when this can happen: + // 1. all values of the field that is the source of copy_to are null + // 2. copy_to points at a field inside a disabled object + // 3. copy_to points at dynamic field which is not yet applied to mapping, we will process it properly on re-parse. + continue; + } + int offset = parent.isRoot() ? 0 : parent.fullPath().length() + 1; + mutableList.add(new IgnoredSourceFieldMapper.NameValue(copyToField, offset, XContentDataHelper.voidValue(), context.doc())); + } + ignoredValuesToWrite = mutableList; + } + + for (NameValue nameValue : ignoredValuesToWrite) { nameValue.doc().add(new StoredField(NAME, encode(nameValue))); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 639c0115150c2..2c851b70d2606 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -12,8 +12,6 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.PrefixCodedTerms; -import org.apache.lucene.index.PrefixCodedTerms.TermIterator; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.intervals.IntervalsSource; @@ -21,12 +19,10 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -561,29 +557,6 @@ protected void checkNoTimeZone(@Nullable ZoneId timeZone) { } } - /** - * Extract a {@link Term} from a query created with {@link #termQuery} by - * recursively removing {@link BoostQuery} wrappers. - * @throws IllegalArgumentException if the wrapped query is not a {@link TermQuery} - */ - public static Term extractTerm(Query termQuery) { - while (termQuery instanceof BoostQuery) { - termQuery = ((BoostQuery) termQuery).getQuery(); - } - if (termQuery instanceof TermInSetQuery tisQuery) { - PrefixCodedTerms terms = tisQuery.getTermData(); - if (terms.size() == 1) { - TermIterator it = terms.iterator(); - BytesRef term = it.next(); - return new Term(it.field(), term); - } - } - if (termQuery instanceof TermQuery == false) { - throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); - } - return ((TermQuery) termQuery).getTerm(); - } - /** * Get the metadata associated with this field. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 0f224c98241db..d18c3283ef909 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -40,7 +40,8 @@ public Set getFeatures() { Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT, SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX, - FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT + FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, + SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index e4ce38d6cec0b..f9c854749e885 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -808,6 +808,42 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } + ObjectMapper findParentMapper(String leafFieldPath) { + var pathComponents = leafFieldPath.split("\\."); + int startPathComponent = 0; + + ObjectMapper current = this; + String pathInCurrent = leafFieldPath; + + while (current != null) { + if (current.mappers.containsKey(pathInCurrent)) { + return current; + } + + // Go one level down if possible + var parent = current; + current = null; + + var childMapperName = new StringBuilder(); + for (int i = startPathComponent; i < pathComponents.length - 1; i++) { + if (childMapperName.isEmpty() == false) { + childMapperName.append("."); + } + childMapperName.append(pathComponents[i]); + + var childMapper = parent.mappers.get(childMapperName.toString()); + if (childMapper instanceof ObjectMapper objectMapper) { + current = objectMapper; + startPathComponent = i + 1; + pathInCurrent = pathInCurrent.substring(childMapperName.length() + 1); + break; + } + } + } + + return null; + } + protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers, boolean isFragment) { var fields = mappers.sorted(Comparator.comparing(Mapper::fullPath)) .map(Mapper::syntheticFieldLoader) @@ -828,10 +864,18 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private final boolean isFragment; + private boolean storedFieldLoadersHaveValues; private boolean docValuesLoadersHaveValues; private boolean ignoredValuesPresent; private List ignoredValues; + // If this loader has anything to write. + // In special cases this can be false even if doc values loaders or stored field loaders + // have values. + // F.e. objects that only contain fields that are destinations of copy_to. + private boolean writersHaveValues; + // Use an ordered map between field names and writers to order writing by field name. + private TreeMap currentWriters; private SyntheticSourceFieldLoader(List fields, boolean isFragment) { this.fields = fields; @@ -882,9 +926,55 @@ public boolean advanceToDoc(int docId) throws IOException { } } + @Override + public void prepare() { + if ((storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent) == false) { + writersHaveValues = false; + return; + } + + for (var loader : fields) { + // Currently this logic is only relevant for object loaders. + if (loader instanceof ObjectMapper.SyntheticSourceFieldLoader objectSyntheticFieldLoader) { + objectSyntheticFieldLoader.prepare(); + } + } + + currentWriters = new TreeMap<>(); + + if (ignoredValues != null && ignoredValues.isEmpty() == false) { + for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { + if (value.hasValue()) { + writersHaveValues |= true; + } + + var existing = currentWriters.get(value.name()); + if (existing == null) { + currentWriters.put(value.name(), new FieldWriter.IgnoredSource(value)); + } else if (existing instanceof FieldWriter.IgnoredSource isw) { + isw.mergeWith(value); + } + } + } + + for (SourceLoader.SyntheticFieldLoader field : fields) { + if (field.hasValue()) { + if (currentWriters.containsKey(field.fieldName()) == false) { + writersHaveValues |= true; + currentWriters.put(field.fieldName(), new FieldWriter.FieldLoader(field)); + } else { + // Skip if the field source is stored separately, to avoid double-printing. + // Make sure to reset the state of loader so that values stored inside will not + // be used after this document is finished. + field.reset(); + } + } + } + } + @Override public boolean hasValue() { - return storedFieldLoadersHaveValues || docValuesLoadersHaveValues || ignoredValuesPresent; + return writersHaveValues; } @Override @@ -892,12 +982,13 @@ public void write(XContentBuilder b) throws IOException { if (hasValue() == false) { return; } + if (isRoot() && isEnabled() == false) { // If the root object mapper is disabled, it is expected to contain // the source encapsulated within a single ignored source value. assert ignoredValues.size() == 1 : ignoredValues.size(); XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); - ignoredValues = null; + softReset(); return; } @@ -907,41 +998,12 @@ public void write(XContentBuilder b) throws IOException { b.startObject(leafName()); } - if (ignoredValues != null && ignoredValues.isEmpty() == false) { - // Use an ordered map between field names and writer functions, to order writing by field name. - Map orderedFields = new TreeMap<>(); - for (IgnoredSourceFieldMapper.NameValue value : ignoredValues) { - var existing = orderedFields.get(value.name()); - if (existing == null) { - orderedFields.put(value.name(), new FieldWriter.IgnoredSource(value)); - } else if (existing instanceof FieldWriter.IgnoredSource isw) { - isw.mergeWith(value); - } - } - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - if (orderedFields.containsKey(field.fieldName()) == false) { - orderedFields.put(field.fieldName(), new FieldWriter.FieldLoader(field)); - } else { - // Skip if the field source is stored separately, to avoid double-printing. - // Make sure to reset the state of loader so that values stored inside will not - // be used after this document is finished. - field.reset(); - } - } - } - - for (var writer : orderedFields.values()) { + for (var writer : currentWriters.values()) { + if (writer.hasValue()) { writer.writeTo(b); } - ignoredValues = null; - } else { - for (SourceLoader.SyntheticFieldLoader field : fields) { - if (field.hasValue()) { - field.write(b); - } - } } + b.endObject(); softReset(); } @@ -957,6 +1019,8 @@ private void softReset() { storedFieldLoadersHaveValues = false; docValuesLoadersHaveValues = false; ignoredValuesPresent = false; + ignoredValues = null; + writersHaveValues = false; } @Override @@ -986,34 +1050,49 @@ public String fieldName() { interface FieldWriter { void writeTo(XContentBuilder builder) throws IOException; + boolean hasValue(); + record FieldLoader(SourceLoader.SyntheticFieldLoader loader) implements FieldWriter { @Override public void writeTo(XContentBuilder builder) throws IOException { loader.write(builder); } + + @Override + public boolean hasValue() { + return loader.hasValue(); + } } class IgnoredSource implements FieldWriter { private final String fieldName; private final String leafName; - private final List values; + private final List encodedValues; IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) { this.fieldName = initialValue.name(); this.leafName = initialValue.getFieldName(); - this.values = new ArrayList<>(); - this.values.add(initialValue.value()); + this.encodedValues = new ArrayList<>(); + if (initialValue.hasValue()) { + this.encodedValues.add(initialValue.value()); + } } @Override public void writeTo(XContentBuilder builder) throws IOException { - XContentDataHelper.writeMerged(builder, leafName, values); + XContentDataHelper.writeMerged(builder, leafName, encodedValues); + } + + @Override + public boolean hasValue() { + return encodedValues.isEmpty() == false; } public FieldWriter mergeWith(IgnoredSourceFieldMapper.NameValue nameValue) { assert Objects.equals(nameValue.name(), fieldName) : "IgnoredSource is merged with wrong field data"; - - values.add(nameValue.value()); + if (nameValue.hasValue()) { + encodedValues.add(nameValue.value()); + } return this; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3318595ed7129..118cdbffc5db9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -48,6 +48,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { "mapper.source.synthetic_source_with_copy_to_and_doc_values_false" ); public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix"); + public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature( + "mapper.source.synthetic_source_copy_to_inside_objects_fix" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index baff3835d104b..ec255a53e7c5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -209,6 +209,9 @@ public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBu if (docValuesLoader != null) { docValuesLoader.advanceToDoc(docId); } + + loader.prepare(); + // TODO accept a requested xcontent type if (loader.hasValue()) { loader.write(b); @@ -299,6 +302,16 @@ public String fieldName() { */ DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException; + /** + Perform any preprocessing needed before producing synthetic source + and deduce whether this mapper (and its children, if any) have values to write. + The expectation is for this method to be called before {@link SyntheticFieldLoader#hasValue()} + and {@link SyntheticFieldLoader#write(XContentBuilder)} are used. + */ + default void prepare() { + // Noop + } + /** * Has this field loaded any values for this document? */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 354f0ec92b0fa..8bacaf8505f91 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -72,7 +72,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE } /** - * Returns a special encoded value that signals that values of this field + * Returns a special encoded value that signals that this field * should not be present in synthetic source. * * An example is a field that has values copied to it using copy_to. @@ -80,7 +80,7 @@ public static BytesRef encodeXContentBuilder(XContentBuilder builder) throws IOE * synthetic _source same as it wouldn't be present in stored source. * @return */ - public static BytesRef nothing() { + public static BytesRef voidValue() { return new BytesRef(new byte[] { VOID_ENCODING }); } @@ -112,41 +112,27 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { /** * Writes encoded values to provided builder. If there are multiple values they are merged into * a single resulting array. + * + * Note that this method assumes all encoded parts have values that need to be written (are not VOID encoded). * @param b destination * @param fieldName name of the field that is written * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened. * @throws IOException */ static void writeMerged(XContentBuilder b, String fieldName, List encodedParts) throws IOException { - var partsWithData = 0; - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - partsWithData++; - } - } - - if (partsWithData == 0) { + if (encodedParts.isEmpty()) { return; } - if (partsWithData == 1) { + if (encodedParts.size() == 1) { b.field(fieldName); - for (BytesRef encodedPart : encodedParts) { - if (isDataPresent(encodedPart)) { - XContentDataHelper.decodeAndWrite(b, encodedPart); - } - } - + XContentDataHelper.decodeAndWrite(b, encodedParts.get(0)); return; } b.startArray(fieldName); for (var encodedValue : encodedParts) { - if (isDataPresent(encodedValue) == false) { - continue; - } - Optional encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) { case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( getXContentType(encodedValue) diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index 55df1f144bb14..aeffff28269dd 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -12,7 +12,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; @@ -23,6 +25,9 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; /** * A Span Query that matches documents containing a term. @@ -77,8 +82,32 @@ protected SpanQuery doToQuery(SearchExecutionContext context) throws IOException if (mapper == null) { term = new Term(fieldName, BytesRefs.toBytesRef(value)); } else { + if (mapper.getTextSearchInfo().hasPositions() == false) { + throw new IllegalArgumentException( + "Span term query requires position data, but field " + fieldName + " was indexed without position data" + ); + } Query termQuery = mapper.termQuery(value, context); - term = MappedFieldType.extractTerm(termQuery); + List termsList = new ArrayList<>(); + termQuery.visit(new QueryVisitor() { + @Override + public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { + if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.FILTER) { + return this; + } + return EMPTY_VISITOR; + } + + @Override + public void consumeTerms(Query query, Term... terms) { + termsList.addAll(Arrays.asList(terms)); + } + }); + if (termsList.size() != 1) { + // This is for safety, but we have called mapper.termQuery above: we really should get one and only one term from the query? + throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); + } + term = termsList.get(0); } return new SpanTermQuery(term); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index c4f0d72c06a80..62d2aa1f026f7 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1514,6 +1514,22 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException { engine.forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(), forceMerge.onlyExpungeDeletes(), forceMerge.forceMergeUUID()); } + public void triggerPendingMerges() throws IOException { + switch (state /* single volatile read */) { + case STARTED, POST_RECOVERY -> getEngine().forceMerge( + // don't immediately flush - if any merging happens then we don't wait for it anyway + false, + // don't apply any segment count limit, we only want to call IndexWriter#maybeMerge + ForceMergeRequest.Defaults.MAX_NUM_SEGMENTS, + // don't look for expunge-delete merges, we only want to call IndexWriter#maybeMerge + false, + // force-merge UUID is not used when calling IndexWriter#maybeMerge + null + ); + // otherwise shard likely closed and maybe reopened, nothing to do + } + } + /** * Creates a new {@link IndexCommit} snapshot from the currently running engine. All resources referenced by this * commit won't be freed until the commit / snapshot is closed. diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 89f651468068d..43bfe7e0e7e2c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -262,6 +262,7 @@ public class IndicesService extends AbstractLifecycleComponent private final TimestampFieldMapperService timestampFieldMapperService; private final CheckedBiConsumer requestCacheKeyDifferentiator; private final MapperMetrics mapperMetrics; + private final PostRecoveryMerger postRecoveryMerger; @Override protected void doStart() { @@ -378,6 +379,8 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon clusterService.getClusterSettings().addSettingsUpdateConsumer(ALLOW_EXPENSIVE_QUERIES, this::setAllowExpensiveQueries); this.timestampFieldMapperService = new TimestampFieldMapperService(settings, threadPool, this); + + this.postRecoveryMerger = new PostRecoveryMerger(settings, threadPool.executor(ThreadPool.Names.FORCE_MERGE), this::getShardOrNull); } private static final String DANGLING_INDICES_UPDATE_THREAD_NAME = "DanglingIndices#updateTask"; @@ -890,23 +893,29 @@ public void createShard( RecoveryState recoveryState = indexService.createRecoveryState(shardRouting, targetNode, sourceNode); IndexShard indexShard = indexService.createShard(shardRouting, globalCheckpointSyncer, retentionLeaseSyncer); indexShard.addShardFailureCallback(onShardFailure); - indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService, (mapping, listener) -> { - assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS - : "mapping update consumer only required by local shards recovery"; - AcknowledgedRequest putMappingRequestAcknowledgedRequest = new PutMappingRequest().setConcreteIndex( - shardRouting.index() - ) - .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid - .source(mapping.source().string(), XContentType.JSON); - // concrete index - no name clash, it uses uuid - client.execute( - featureService.clusterHasFeature(clusterService.state(), SUPPORTS_AUTO_PUT) - ? TransportAutoPutMappingAction.TYPE - : TransportPutMappingAction.TYPE, - putMappingRequestAcknowledgedRequest.ackTimeout(TimeValue.MAX_VALUE).masterNodeTimeout(TimeValue.MAX_VALUE), - new RefCountAwareThreadedActionListener<>(threadPool.generic(), listener.map(ignored -> null)) - ); - }, this, clusterStateVersion); + indexShard.startRecovery( + recoveryState, + recoveryTargetService, + postRecoveryMerger.maybeMergeAfterRecovery(shardRouting, recoveryListener), + repositoriesService, + (mapping, listener) -> { + assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS + : "mapping update consumer only required by local shards recovery"; + AcknowledgedRequest putMappingRequestAcknowledgedRequest = new PutMappingRequest() + // concrete index - no name clash, it uses uuid + .setConcreteIndex(shardRouting.index()) + .source(mapping.source().string(), XContentType.JSON); + client.execute( + featureService.clusterHasFeature(clusterService.state(), SUPPORTS_AUTO_PUT) + ? TransportAutoPutMappingAction.TYPE + : TransportPutMappingAction.TYPE, + putMappingRequestAcknowledgedRequest.ackTimeout(TimeValue.MAX_VALUE).masterNodeTimeout(TimeValue.MAX_VALUE), + new RefCountAwareThreadedActionListener<>(threadPool.generic(), listener.map(ignored -> null)) + ); + }, + this, + clusterStateVersion + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java new file mode 100644 index 0000000000000..11c8a2b4d6048 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.indices; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardLongFieldRange; +import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; +import org.elasticsearch.indices.recovery.RecoveryFailedException; +import org.elasticsearch.indices.recovery.RecoveryState; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.concurrent.Executor; +import java.util.function.Function; + +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_HOT_NODE_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_ROLE; +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.INDEX_ROLE; + +/** + * Triggers a check for pending merges when a shard completes recovery. + */ +class PostRecoveryMerger { + + private static final Logger logger = LogManager.getLogger(PostRecoveryMerger.class); + + private static final boolean TRIGGER_MERGE_AFTER_RECOVERY; + + static { + final var propertyValue = System.getProperty("es.trigger_merge_after_recovery"); + if (propertyValue == null) { + TRIGGER_MERGE_AFTER_RECOVERY = true; + } else if ("false".equals(propertyValue)) { + TRIGGER_MERGE_AFTER_RECOVERY = false; + } else { + throw new IllegalStateException( + "system property [es.trigger_merge_after_recovery] may only be set to [false], but was [" + propertyValue + "]" + ); + } + } + + /** + * Throttled runner to avoid multiple concurrent calls to {@link IndexWriter#maybeMerge()}: we do not need to execute these things + * especially quickly, as long as they happen eventually, and each such call may involve some IO (reading the soft-deletes doc values to + * count deleted docs). Note that we're not throttling any actual merges, just the checks to see what merges might be needed. Throttling + * merges across shards is a separate issue, but normally this mechanism won't trigger any new merges anyway. + */ + private final ThrottledTaskRunner postRecoveryMergeRunner; + + private final Function shardFunction; + private final boolean enabled; + + PostRecoveryMerger(Settings settings, Executor executor, Function shardFunction) { + this.postRecoveryMergeRunner = new ThrottledTaskRunner(getClass().getCanonicalName(), 1, executor); + this.shardFunction = shardFunction; + this.enabled = + // enabled globally ... + TRIGGER_MERGE_AFTER_RECOVERY + // ... and we are a node that expects nontrivial amounts of indexing work + && (DiscoveryNode.hasRole(settings, DATA_HOT_NODE_ROLE) + || DiscoveryNode.hasRole(settings, DATA_CONTENT_NODE_ROLE) + || DiscoveryNode.hasRole(settings, DATA_ROLE) + || DiscoveryNode.hasRole(settings, INDEX_ROLE)); + } + + PeerRecoveryTargetService.RecoveryListener maybeMergeAfterRecovery( + ShardRouting shardRouting, + PeerRecoveryTargetService.RecoveryListener recoveryListener + ) { + if (enabled == false) { + return recoveryListener; + } + + if (shardRouting.isPromotableToPrimary() == false) { + return recoveryListener; + } + + final var shardId = shardRouting.shardId(); + return new PeerRecoveryTargetService.RecoveryListener() { + @Override + public void onRecoveryDone( + RecoveryState state, + ShardLongFieldRange timestampMillisFieldRange, + ShardLongFieldRange eventIngestedMillisFieldRange + ) { + postRecoveryMergeRunner.enqueueTask(new PostRecoveryMerge(shardId)); + recoveryListener.onRecoveryDone(state, timestampMillisFieldRange, eventIngestedMillisFieldRange); + } + + @Override + public void onRecoveryFailure(RecoveryFailedException e, boolean sendShardFailure) { + recoveryListener.onRecoveryFailure(e, sendShardFailure); + } + }; + } + + class PostRecoveryMerge implements ActionListener { + private final ShardId shardId; + + PostRecoveryMerge(ShardId shardId) { + this.shardId = shardId; + } + + @Override + public void onResponse(Releasable releasable) { + try (releasable) { + final var indexShard = shardFunction.apply(shardId); + if (indexShard == null) { + return; + } + + indexShard.triggerPendingMerges(); + } catch (Exception e) { + logFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + logFailure(e); + } + + private void logFailure(Exception e) { + // post-recovery merge is a best-effort thing, failure needs no special handling + logger.debug(() -> Strings.format("failed to execute post-recovery merge of [%s]", shardId), e); + } + } +} diff --git a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java b/server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java similarity index 94% rename from libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java rename to server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java index be3e76bd83396..d9bf0d653bb62 100644 --- a/libs/plugin-classloader/src/main/java/org/elasticsearch/plugins/loader/ExtendedPluginsClassLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/ExtendedPluginsClassLoader.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.plugins.loader; +package org.elasticsearch.plugins; import java.security.AccessController; import java.security.PrivilegedAction; @@ -17,7 +17,7 @@ /** * A classloader that is a union over the parent core classloader and classloaders of extended plugins. */ -public class ExtendedPluginsClassLoader extends ClassLoader { +class ExtendedPluginsClassLoader extends ClassLoader { /** Loaders of plugins extended by a plugin. */ private final List extendedLoaders; diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java deleted file mode 100644 index a5ca26283231d..0000000000000 --- a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.plugins; - -import org.elasticsearch.plugins.loader.ExtendedPluginsClassLoader; - -import java.util.List; - -// TODO: remove this indirection now that transport client is gone -/** - * This class exists solely as an intermediate layer to avoid causing PluginsService - * to load ExtendedPluginsClassLoader when used in the transport client. - */ -class PluginLoaderIndirection { - - static ClassLoader createLoader(ClassLoader parent, List extendedLoaders) { - return ExtendedPluginsClassLoader.create(parent, extendedLoaders); - } -} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 47f0a50ff309c..d5dd6d62d615e 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -468,7 +468,7 @@ private void loadBundle( ); } - final ClassLoader parentLoader = PluginLoaderIndirection.createLoader( + final ClassLoader parentLoader = ExtendedPluginsClassLoader.create( getClass().getClassLoader(), extendedPlugins.stream().map(LoadedPlugin::loader).toList() ); diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index e48677f46d57a..17eda305b5ccf 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -105,19 +105,19 @@ public boolean isContentConsumed() { protected RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel ) { - this(parserConfig, params, path, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); + this(parserConfig, params, rawPath, headers, httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); } @SuppressWarnings("this-escape") private RestRequest( XContentParserConfiguration parserConfig, Map params, - String path, + String rawPath, Map> headers, HttpRequest httpRequest, HttpChannel httpChannel, @@ -149,7 +149,7 @@ private RestRequest( : parserConfig.withRestApiVersion(effectiveApiVersion); this.httpChannel = httpChannel; this.params = params; - this.rawPath = path; + this.rawPath = rawPath; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; } @@ -204,11 +204,10 @@ void ensureSafeBuffers() { */ public static RestRequest request(XContentParserConfiguration parserConfig, HttpRequest httpRequest, HttpChannel httpChannel) { Map params = params(httpRequest.uri()); - String path = path(httpRequest.uri()); return new RestRequest( parserConfig, params, - path, + httpRequest.rawPath(), httpRequest.getHeaders(), httpRequest, httpChannel, @@ -229,15 +228,6 @@ private static Map params(final String uri) { return params; } - private static String path(final String uri) { - final int index = uri.indexOf('?'); - if (index >= 0) { - return uri.substring(0, index); - } else { - return uri; - } - } - /** * Creates a new REST request. The path is not decoded so this constructor will not throw a * {@link BadParameterException}. diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index ff87bb834f3e1..33ee87c0c0b5a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.bulk.IncrementalBulkService; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; import org.elasticsearch.common.bytes.ReleasableBytesReference; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -59,13 +61,16 @@ public class RestBulkAction extends BaseRestHandler { public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in bulk requests is deprecated."; + public static final String FAILURE_STORE_STATUS_CAPABILITY = "failure_store_status"; private final boolean allowExplicitIndex; private final IncrementalBulkService bulkHandler; + private final Set capabilities; public RestBulkAction(Settings settings, IncrementalBulkService bulkHandler) { this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); this.bulkHandler = bulkHandler; + this.capabilities = DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(FAILURE_STORE_STATUS_CAPABILITY) : Set.of(); } @Override @@ -291,4 +296,9 @@ public boolean supportsBulkContent() { public boolean allowsUnsafeBuffers() { return true; } + + @Override + public Set supportedCapabilities() { + return capabilities; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 4181bd63eb60b..2eb0e5a1ef038 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; @@ -26,15 +27,20 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.action.document.RestBulkAction.FAILURE_STORE_STATUS_CAPABILITY; @ServerlessScope(Scope.PUBLIC) public class RestIndexAction extends BaseRestHandler { static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " + "or /{index}/_create/{id})."; + private final Set capabilities = DataStream.isFailureStoreFeatureFlagEnabled() + ? Set.of(FAILURE_STORE_STATUS_CAPABILITY) + : Set.of(); @Override public List routes() { @@ -144,4 +150,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); } + @Override + public Set supportedCapabilities() { + return capabilities; + } } diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java index d23bbec71a01d..b14116b3d55ba 100644 --- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java +++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; @@ -602,6 +603,9 @@ public void onResponse(T t) { @Override public void onFailure(Exception e) { Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus) { + cause = cause.getCause(); + } if (cause instanceof IndexNotFoundException) { delegate.onFailure(new ResourceNotFoundException("synonyms set [" + synonymSetId + "] not found")); return; diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 756e8106f631a..55abdc84fc8fb 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -53,11 +53,6 @@ grant codeBase "${codebase.lucene-misc}" { permission java.nio.file.LinkPermission "hard"; }; -grant codeBase "${codebase.elasticsearch-plugin-classloader}" { - // needed to create the classloader which allows plugins to extend other plugins - permission java.lang.RuntimePermission "createClassLoader"; -}; - grant codeBase "${codebase.elasticsearch-core}" { permission java.lang.RuntimePermission "createClassLoader"; permission java.lang.RuntimePermission "getClassLoader"; diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 8e05c05f59df9..31739850e2d35 100644 --- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.TimestampParsingException; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.VersionMismatchException; @@ -837,6 +838,7 @@ public void testIds() { ids.put(180, PersistentTaskNodeNotAssignedException.class); ids.put(181, ResourceAlreadyUploadedException.class); ids.put(182, IngestPipelineException.class); + ids.put(183, IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class); Map, Integer> reverse = new HashMap<>(); for (Map.Entry> entry : ids.entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java index f7fc248376e47..e3ab7463ad941 100644 --- a/server/src/test/java/org/elasticsearch/TransportVersionTests.java +++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -199,4 +200,40 @@ public void testToString() { assertEquals("2000099", TransportVersion.fromId(2_00_00_99).toString()); assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString()); } + + /** + * Until 9.0 bumps its transport version to 9_000_00_0, all transport changes must be backported to 8.x. + * This test ensures transport versions are dense, so that we have confidence backports have not been missed. + * Note that it does not ensure patches are not missed, but it should catch the majority of misordered + * or missing transport versions. + */ + public void testDenseTransportVersions() { + Set missingVersions = new TreeSet<>(); + TransportVersion previous = null; + for (var tv : TransportVersions.getAllVersions()) { + if (tv.before(TransportVersions.V_8_14_0)) { + continue; + } + if (previous == null) { + previous = tv; + continue; + } + + if (previous.id() + 1000 < tv.id()) { + int nextId = previous.id(); + do { + nextId = (nextId + 1000) / 1000 * 1000; + missingVersions.add(nextId); + } while (nextId + 1000 < tv.id()); + } + previous = tv; + } + if (missingVersions.isEmpty() == false) { + StringBuilder msg = new StringBuilder("Missing transport versions:\n"); + for (Integer id : missingVersions) { + msg.append(" " + id + "\n"); + } + fail(msg.toString()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java index 6f10b003bb59b..5a71473e9b0ed 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java @@ -378,6 +378,8 @@ public void testFailingEntireShardRedirectsToFailureStore() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("Could not find redirected item")); assertThat(failedItem, is(notNullValue())); + // Ensure the status in the successful response gets through + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED)); } /** @@ -403,6 +405,8 @@ public void testFailingDocumentRedirectsToFailureStore() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("Could not find redirected item")); assertThat(failedItem.getIndex(), is(notNullValue())); + // Ensure the status in the successful response gets through + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED)); } /** @@ -441,6 +445,7 @@ public void testFailureStoreShardFailureRejectsDocument() throws Exception { assertThat(failedItem.getFailure().getCause().getSuppressed().length, is(not(equalTo(0)))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0], is(instanceOf(MapperException.class))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0].getMessage(), is(equalTo("failure store test failure"))); + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.FAILED)); } /** @@ -476,6 +481,7 @@ public void testFailedDocumentCanNotBeConvertedFails() throws Exception { assertThat(failedItem.getFailure().getCause().getSuppressed().length, is(not(equalTo(0)))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0], is(instanceOf(IOException.class))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0].getMessage(), is(equalTo("Could not serialize json"))); + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.FAILED)); } /** @@ -565,6 +571,8 @@ public void testRetryableBlockAcceptsFailureStoreDocument() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("Could not find redirected item")); assertThat(failedItem, is(notNullValue())); + // Ensure the status in the successful response gets through + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED)); verify(observer, times(1)).isTimedOut(); verify(observer, times(1)).waitForNextChange(any()); @@ -617,6 +625,7 @@ public void testBlockedClusterRejectsFailureStoreDocument() throws Exception { failedItem.getFailure().getCause().getSuppressed()[0].getMessage(), is(equalTo("blocked by: [FORBIDDEN/5/index read-only (api)];")) ); + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.FAILED)); verify(observer, times(0)).isTimedOut(); verify(observer, times(0)).waitForNextChange(any()); @@ -677,6 +686,7 @@ public void testOperationTimeoutRejectsFailureStoreDocument() throws Exception { failedItem.getFailure().getCause().getSuppressed()[0].getMessage(), is(equalTo("blocked by: [SERVICE_UNAVAILABLE/2/no master];")) ); + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.FAILED)); verify(observer, times(2)).isTimedOut(); verify(observer, times(1)).waitForNextChange(any()); @@ -779,6 +789,8 @@ public void testLazilyRollingOverFailureStore() throws Exception { .findFirst() .orElseThrow(() -> new AssertionError("Could not find redirected item")); assertThat(failedItem, is(notNullValue())); + // Ensure the status in the successful response gets through + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.USED)); } /** @@ -826,6 +838,7 @@ public void testFailureWhileRollingOverFailureStore() throws Exception { assertThat(failedItem.getFailure().getCause().getSuppressed().length, is(not(equalTo(0)))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0], is(instanceOf(Exception.class))); assertThat(failedItem.getFailure().getCause().getSuppressed()[0].getMessage(), is(equalTo("rollover failed"))); + assertThat(failedItem.getFailureStoreStatus(), equalTo(IndexDocFailureStoreStatus.FAILED)); } /** @@ -839,14 +852,12 @@ private static BiConsumer> a * Accepts all write operations from the given request object when it is encountered in the mock shard bulk action */ private static BiConsumer> acceptAllShardWrites() { - return (BulkShardRequest request, ActionListener listener) -> { - listener.onResponse( - new BulkShardResponse( - request.shardId(), - Arrays.stream(request.items()).map(item -> requestToResponse(request.shardId(), item)).toArray(BulkItemResponse[]::new) - ) - ); - }; + return (BulkShardRequest request, ActionListener listener) -> listener.onResponse( + new BulkShardResponse( + request.shardId(), + Arrays.stream(request.items()).map(item -> requestToResponse(request.shardId(), item)).toArray(BulkItemResponse[]::new) + ) + ); } /** @@ -923,8 +934,11 @@ private BiConsumer> thatFail * Create a shard-level result given a bulk item */ private static BulkItemResponse requestToResponse(ShardId shardId, BulkItemRequest itemRequest) { + var failureStatus = itemRequest.request() instanceof IndexRequest ir && ir.isWriteToFailureStore() + ? IndexDocFailureStoreStatus.USED + : IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN; return BulkItemResponse.success(itemRequest.id(), itemRequest.request().opType(), switch (itemRequest.request().opType()) { - case INDEX, CREATE -> new IndexResponse(shardId, itemRequest.request().id(), 1, 1, 1, true); + case INDEX, CREATE -> new IndexResponse(shardId, itemRequest.request().id(), 1, 1, 1, true, null, failureStatus); case UPDATE -> new UpdateResponse(shardId, itemRequest.request().id(), 1, 1, 1, DocWriteResponse.Result.UPDATED); case DELETE -> new DeleteResponse(shardId, itemRequest.request().id(), 1, 1, 1, true); }); diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index fb357d3dda414..32297e0c09b8f 100644 --- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.metadata.DataStreamAlias; @@ -132,7 +133,17 @@ public void testIndexResponse() { String id = randomAlphaOfLengthBetween(3, 10); long version = randomLong(); boolean created = randomBoolean(); - IndexResponse indexResponse = new IndexResponse(shardId, id, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, version, created); + var failureStatus = randomFrom(Set.of(IndexDocFailureStoreStatus.USED, IndexDocFailureStoreStatus.NOT_APPLICABLE_OR_UNKNOWN)); + IndexResponse indexResponse = new IndexResponse( + shardId, + id, + SequenceNumbers.UNASSIGNED_SEQ_NO, + 0, + version, + created, + null, + failureStatus + ); int total = randomIntBetween(1, 10); int successful = randomIntBetween(1, 10); ReplicationResponse.ShardInfo shardInfo = ReplicationResponse.ShardInfo.of(total, successful); @@ -149,6 +160,7 @@ public void testIndexResponse() { assertEquals(total, indexResponse.getShardInfo().getTotal()); assertEquals(successful, indexResponse.getShardInfo().getSuccessful()); assertEquals(forcedRefresh, indexResponse.forcedRefresh()); + assertEquals(failureStatus, indexResponse.getFailureStoreStatus()); Object[] args = new Object[] { shardId.getIndexName(), id, @@ -157,10 +169,11 @@ public void testIndexResponse() { SequenceNumbers.UNASSIGNED_SEQ_NO, 0, total, - successful }; + successful, + failureStatus.getLabel() }; assertEquals(Strings.format(""" IndexResponse[index=%s,id=%s,version=%s,result=%s,seqNo=%s,primaryTerm=%s,shards=\ - {"total":%s,"successful":%s,"failed":0}]\ + {"total":%s,"successful":%s,"failed":0},failure_store=%s]\ """, args), indexResponse.toString()); } diff --git a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java index 68a161d426f48..82463d601d164 100644 --- a/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/RankFeaturePhaseTests.java @@ -536,7 +536,7 @@ public void sendExecuteRankFeature( // override the RankFeaturePhase to raise an exception RankFeaturePhase rankFeaturePhase = new RankFeaturePhase(results, null, mockSearchPhaseContext, null) { @Override - void innerRun() { + void innerRun(RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext) { throw new IllegalArgumentException("simulated failure"); } @@ -1142,7 +1142,13 @@ public void moveToNextPhase( ) { // this is called after the RankFeaturePhaseCoordinatorContext has been executed phaseDone.set(true); - finalResults[0] = reducedQueryPhase.sortedTopDocs().scoreDocs(); + try { + finalResults[0] = reducedQueryPhase == null + ? queryPhaseResults.reduce().sortedTopDocs().scoreDocs() + : reducedQueryPhase.sortedTopDocs().scoreDocs(); + } catch (Exception e) { + throw new AssertionError(e); + } logger.debug("Skipping moving to next phase"); } }; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index 81c19946753db..bbcf1ca33a0c2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -244,8 +244,8 @@ public void testCreateDataStreamWithFailureStoreInitialized() throws Exception { ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -284,8 +284,8 @@ public void testCreateDataStreamWithFailureStoreUninitialized() throws Exception ActionListener.noop(), false ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); @@ -321,8 +321,8 @@ public void testCreateDataStreamWithFailureStoreWithRefreshRate() throws Excepti ActionListener.noop(), true ); - var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); - var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.startTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.startTime()); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); assertThat(newState.metadata().index(backingIndexName), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 78e900898bfc9..498c04c005304 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -27,7 +28,6 @@ import org.elasticsearch.cluster.LocalMasterServiceTask; import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.SimpleBatchedExecutor; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.coordination.ClusterStatePublisher; import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; @@ -1571,7 +1571,7 @@ public void onAckFailure(Exception e) { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(TimeValue.ZERO, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), null) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -1623,7 +1623,7 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, null), null) { + new AckedClusterStateUpdateTask(ackedRequest(ackTimeout, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), null) { @Override public ClusterState execute(ClusterState currentState) { threadPool.getThreadContext().addResponseHeader(responseHeaderName, responseHeaderValue); @@ -1678,7 +1678,10 @@ public void onAckTimeout() { masterService.submitUnbatchedStateUpdateTask( "test2", - new AckedClusterStateUpdateTask(ackedRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT, null), null) { + new AckedClusterStateUpdateTask( + ackedRequest(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT, MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT), + null + ) { @Override public ClusterState execute(ClusterState currentState) { return ClusterState.builder(currentState).build(); @@ -2657,20 +2660,15 @@ public static ClusterState discoveryState(MasterService masterService) { } /** - * Returns a plain {@link AckedRequest} that does not implement any functionality outside of the timeout getters. + * Returns a plain {@link AcknowledgedRequest} that does not implement any functionality outside of the timeout getters. */ - public static AckedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { - return new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return ackTimeout; - } - - @Override - public TimeValue masterNodeTimeout() { - return masterNodeTimeout; + public static AcknowledgedRequest ackedRequest(TimeValue ackTimeout, TimeValue masterNodeTimeout) { + class BareAcknowledgedRequest extends AcknowledgedRequest { + BareAcknowledgedRequest() { + super(masterNodeTimeout, ackTimeout); } - }; + } + return new BareAcknowledgedRequest(); } /** diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java index 86c268dd2a092..e05b67874ddbb 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisTests.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.List; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.is; public class AnalysisTests extends ESTestCase { @@ -104,4 +105,92 @@ public void testParseWordList() throws IOException { List wordList = Analysis.getWordList(env, nodeSettings, "foo.bar"); assertEquals(Arrays.asList("hello", "world"), wordList); } + + public void testParseDuplicates() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "true") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra stuff that gets discarded"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + List wordList = Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", true, true); + assertEquals(List.of("最終契約,最終契約,最終契約,カスタム名 詞"), wordList); + } + + public void testFailOnDuplicates() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "false") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契,最終契,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, + () -> Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", false, true) + ); + assertThat(exc.getMessage(), containsString("[最終契約] in user dictionary at line [5]")); + } + + public void testParseDuplicatesWComments() throws IOException { + Path tempDir = createTempDir(); + Path dict = tempDir.resolve("foo.dict"); + Settings nodeSettings = Settings.builder() + .put("foo.path", tempDir.resolve(dict)) + .put("bar.list", "") + .put("soup.lenient", "true") + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .build(); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞"); + writer.write('\n'); + writer.write("# This is a test of the emergency broadcast system"); + writer.write('\n'); + writer.write("最終契約,最終契約,最終契約,カスタム名 詞,extra"); + writer.write('\n'); + } + Environment env = TestEnvironment.newEnvironment(nodeSettings); + List wordList = Analysis.getWordList(env, nodeSettings, "foo.path", "bar.list", "soup.lenient", false, true); + assertEquals( + List.of( + "# This is a test of the emergency broadcast system", + "最終契約,最終契約,最終契約,カスタム名 詞", + "# This is a test of the emergency broadcast system" + ), + wordList + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 0e97d2b3b46ae..eaa7bf6528203 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -1518,6 +1518,37 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx {"path":{"at":{"foo":"A"}}}""", syntheticSource); } + public void testCopyToLogicInsideObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path"); + b.startObject("properties"); + { + b.startObject("at").field("type", "keyword").field("copy_to", "copy_top.copy").endObject(); + } + b.endObject(); + b.endObject(); + b.startObject("copy_top"); + b.startObject("properties"); + { + b.startObject("copy").field("type", "keyword").endObject(); + } + b.endObject(); + b.endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("path"); + b.field("at", "A"); + b.endObject(); + }; + + var doc = documentMapper.parse(source(document)); + assertNotNull(doc.docs().get(0).getField("copy_top.copy")); + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"path\":{\"at\":\"A\"}}", syntheticSource); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 8ba57824cf434..3312c94e8a0e1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -811,4 +811,40 @@ public void testFlattenExplicitSubobjectsTrue() { exception.getMessage() ); } + + public void testFindParentMapper() { + MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); + + var rootBuilder = new RootObjectMapper.Builder("_doc", Optional.empty()); + rootBuilder.add(new KeywordFieldMapper.Builder("keyword", IndexVersion.current())); + + var child = new ObjectMapper.Builder("child", Optional.empty()); + child.add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())); + child.add(new KeywordFieldMapper.Builder("keyword.with.dot", IndexVersion.current())); + var secondLevelChild = new ObjectMapper.Builder("child2", Optional.empty()); + secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", IndexVersion.current())); + child.add(secondLevelChild); + rootBuilder.add(child); + + var childWithDot = new ObjectMapper.Builder("childwith.dot", Optional.empty()); + childWithDot.add(new KeywordFieldMapper.Builder("keyword3", IndexVersion.current())); + childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", IndexVersion.current())); + rootBuilder.add(childWithDot); + + RootObjectMapper root = rootBuilder.build(rootContext); + + assertEquals("_doc", root.findParentMapper("keyword").fullPath()); + assertNull(root.findParentMapper("aa")); + + assertEquals("child", root.findParentMapper("child.keyword2").fullPath()); + assertEquals("child", root.findParentMapper("child.keyword.with.dot").fullPath()); + assertNull(root.findParentMapper("child.long")); + assertNull(root.findParentMapper("child.long.hello")); + assertEquals("child.child2", root.findParentMapper("child.child2.keyword22").fullPath()); + + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword3").fullPath()); + assertEquals("childwith.dot", root.findParentMapper("childwith.dot.keyword4.with.dot").fullPath()); + assertNull(root.findParentMapper("childwith.dot.long")); + assertNull(root.findParentMapper("childwith.dot.long.hello")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index 4de02178beec1..f4e114da1fa51 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -240,64 +240,6 @@ private void testWriteMergedWithMixedValues(Object value, List multipleV assertEquals(expected, map.get("foo")); } - public void testWriteMergedWithVoidValue() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged(destination, "field", List.of(XContentDataHelper.nothing())); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMultipleVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.nothing(), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{}", Strings.toString(destination)); - } - - public void testWriteMergedWithMixedVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(34); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":34}", Strings.toString(destination)); - } - - public void testWriteMergedWithArraysAndVoidValues() throws IOException { - var destination = XContentFactory.contentBuilder(XContentType.JSON); - destination.startObject(); - - var value = XContentFactory.contentBuilder(XContentType.JSON).value(List.of(3, 4)); - XContentDataHelper.writeMerged( - destination, - "field", - List.of(XContentDataHelper.nothing(), XContentDataHelper.encodeXContentBuilder(value), XContentDataHelper.nothing()) - ); - - destination.endObject(); - - assertEquals("{\"field\":[3,4]}", Strings.toString(destination)); - } - private Map executeWriteMergedOnRepeated(Object value) throws IOException { return executeWriteMergedOnTwoEncodedValues(value, value); } diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java index 5e01f6c304efd..f0f23d8539e13 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java @@ -11,10 +11,12 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.SpanTermQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xcontent.json.JsonStringEncoder; @@ -124,4 +126,22 @@ public void testWithMetadataField() throws IOException { assertEquals(expected, query); } } + + public void testWithBoost() throws IOException { + SearchExecutionContext context = createSearchExecutionContext(); + for (String field : new String[] { "field1", "field2" }) { + SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto"); + spanTermQueryBuilder.boost(10); + Query query = spanTermQueryBuilder.toQuery(context); + Query expected = new BoostQuery(new SpanTermQuery(new Term(field, "toto")), 10); + assertEquals(expected, query); + } + } + + public void testFieldWithoutPositions() { + SearchExecutionContext context = createSearchExecutionContext(); + SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(IdFieldMapper.NAME, "1234"); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> spanTermQueryBuilder.toQuery(context)); + assertEquals("Span term query requires position data, but field _id was indexed without position data", iae.getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java index b3efff4323c20..5862e1bd1329f 100644 --- a/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/rank/RankFeatureShardPhaseTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamOutput; @@ -170,7 +171,12 @@ public RankShardResult buildRankFeatureShardResult(SearchHits hits, int shardId) // no work to be done on the coordinator node for the rank feature phase @Override public RankFeaturePhaseRankCoordinatorContext buildRankFeaturePhaseCoordinatorContext(int size, int from, Client client) { - return null; + return new RankFeaturePhaseRankCoordinatorContext(size, from, DEFAULT_RANK_WINDOW_SIZE) { + @Override + protected void computeScores(RankFeatureDoc[] featureDocs, ActionListener scoreListener) { + throw new AssertionError("not expected"); + } + }; } @Override diff --git a/settings.gradle b/settings.gradle index ab87861105156..2926a9a303375 100644 --- a/settings.gradle +++ b/settings.gradle @@ -91,6 +91,7 @@ List projects = [ 'distribution:tools:keystore-cli', 'distribution:tools:geoip-cli', 'distribution:tools:ansi-console', + 'distribution:tools:entitlement-agent', 'server', 'test:framework', 'test:fixtures:azure-fixture', diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index ce843fc3e15ee..81e120511a40f 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -104,9 +104,12 @@ public DataSourceResponse.ObjectArrayGenerator accept(DataSourceHandler handler) } } - record LeafMappingParametersGenerator(String fieldName, FieldType fieldType, Set eligibleCopyToFields) - implements - DataSourceRequest { + record LeafMappingParametersGenerator( + String fieldName, + FieldType fieldType, + Set eligibleCopyToFields, + DynamicMapping dynamicMapping + ) implements DataSourceRequest { public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 86cf071ce8696..89850cd56bbd0 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -48,7 +49,11 @@ private Supplier> keywordMapping( // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields. // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail. // Actual value being copied does not influence the core logic of copy_to anyway. - if (ESTestCase.randomDouble() <= 0.05) { + // + // TODO + // We don't use copy_to on fields that are inside an object with dynamic: strict + // because we'll hit https://github.com/elastic/elasticsearch/issues/113049. + if (request.dynamicMapping() != DynamicMapping.FORBIDDEN && ESTestCase.randomDouble() <= 0.05) { var options = request.eligibleCopyToFields() .stream() .filter(f -> f.equals(request.fieldName()) == false) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index c6d9d94d61892..ba03b2f91c53c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -62,7 +62,8 @@ List generateChildFields(DynamicMapping dynamicMapping) { new DataSourceRequest.LeafMappingParametersGenerator( fieldName, fieldTypeInfo.fieldType(), - context.getEligibleCopyToDestinations() + context.getEligibleCopyToDestinations(), + dynamicMapping ) ); var generator = fieldTypeInfo.fieldType() diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java index 7b1bf4a77dc72..57e3ce3ce2a86 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/PredefinedField.java @@ -21,7 +21,7 @@ public interface PredefinedField { FieldDataGenerator generator(DataSource dataSource); - record WithType(String fieldName, FieldType fieldType) implements PredefinedField { + record WithType(String fieldName, FieldType fieldType, DynamicMapping dynamicMapping) implements PredefinedField { @Override public String name() { return fieldName; @@ -31,7 +31,7 @@ public String name() { public FieldDataGenerator generator(DataSource dataSource) { // copy_to currently not supported for predefined fields, use WithGenerator if needed var mappingParametersGenerator = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of()) + new DataSourceRequest.LeafMappingParametersGenerator(fieldName, fieldType, Set.of(), dynamicMapping) ); return fieldType().generator(fieldName, dataSource, mappingParametersGenerator); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java index 1072e6ee4c899..53247d6428bfb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java @@ -9,11 +9,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Priority; import org.elasticsearch.core.Nullable; @@ -36,7 +36,7 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask { public static AckedClusterStateUpdateTask wrap( OperationModeUpdateTask task, - AckedRequest request, + AcknowledgedRequest request, ActionListener listener ) { return new AckedClusterStateUpdateTask(task.priority(), request, listener) { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json index 27262507518d2..2bf7607e86d32 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-es-mb.json @@ -2466,6 +2466,104 @@ } } }, + "indexing_pressure": { + "properties": { + "memory": { + "properties": { + "current": { + "properties": { + "all": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "combined_coordinating_and_primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "coordinating": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "replica": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "limit_in_bytes": { + "type": "long" + }, + "total": { + "properties": { + "all": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "combined_coordinating_and_primary": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "coordinating": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + }, + "primary": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + }, + "replica": { + "properties": { + "bytes": { + "type": "long" + }, + "rejections": { + "type": "long" + } + } + } + } + } + } + } + } + }, "indices_stats": { "properties": { "_all": { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index b5ae35bfc8d7f..5e0e625abb914 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -46,6 +46,7 @@ import java.util.Objects; import java.util.function.IntFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -325,7 +326,11 @@ private static void checkState(boolean condition, String msg) { @Override public String toString() { - return this.getClass().getSimpleName() + "[" + "aggregators=" + aggregatorFactories + "]"; + String aggregatorDescriptions = aggregatorFactories.stream() + .map(factory -> "\"" + factory.describe() + "\"") + .collect(Collectors.joining(", ")); + + return this.getClass().getSimpleName() + "[" + "aggregators=[" + aggregatorDescriptions + "]]"; } record SegmentID(int shardIndex, int segmentIndex) { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 44550c62bd7c5..2a882550b67b1 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -323,6 +323,35 @@ public void testProfile() throws IOException { ); } + public void testProfileOrdinalsGroupingOperator() throws IOException { + indexTimestampData(1); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | STATS AVG(value) BY test.keyword"); + builder.profile(true); + if (Build.current().isSnapshot()) { + // Lock to shard level partitioning, so we get consistent profile output + builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); + } + Map result = runEsql(builder); + + List> signatures = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add((String) o.get("operator")); + } + signatures.add(sig); + } + + assertThat(signatures.get(0).get(2), equalTo("OrdinalsGroupingOperator[aggregators=[\"sum of longs\", \"count\"]]")); + } + public void testInlineStatsProfile() throws IOException { assumeTrue("INLINESTATS only available on snapshots", Build.current().isSnapshot()); indexTimestampData(1); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index 996b2b5030d82..9177fcbcd2afb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -261,3 +261,23 @@ warning:Line 1:38: java.lang.IllegalArgumentException: single-value function enc a:integer | b:integer | same:boolean ; + +caseOnTheValue_NotOnTheField +required_capability: fixed_wrong_is_not_null_check_on_case + +FROM employees +| WHERE emp_no < 10022 AND emp_no > 10012 +| KEEP languages, emp_no +| EVAL eval = CASE(languages == 1, null, languages == 2, "bilingual", languages > 2, "multilingual", languages IS NULL, "languages is null") +| SORT languages, emp_no +| WHERE eval IS NOT NULL; + +languages:integer| emp_no:integer|eval:keyword +2 |10016 |bilingual +2 |10017 |bilingual +2 |10018 |bilingual +5 |10014 |multilingual +5 |10015 |multilingual +null |10020 |languages is null +null |10021 |languages is null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f9d83641ab4bd..a2686dfbe19a7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -367,6 +367,80 @@ date1:date | dd_ms:integer 2023-12-02T11:00:00.000Z | 1000 ; +evalDateDiffMonthAsWhole0Months + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | msecs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 + +; + +evalDateDiffMonthAsWhole1Month + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-02-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), months=DATE_DIFF("month", from, to) +; + + from:date | to:date | secs:integer| months:integer +2023-12-31T23:59:59.999Z|2024-02-01T00:00:00.000Z|2678400 |1 + +; + +evalDateDiffYearAsWhole0Years +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2024-01-01T00:00:00"::DATETIME +| EVAL msecs=DATE_DIFF("milliseconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | msecs:integer | years:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|1 |0 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2023-12-31T23:59:59.999Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2023-12-31T23:59:59.999Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearAsWhole1Year +required_capability: date_diff_year_calendarial + +ROW from="2024-01-01T00:00:00Z"::DATETIME, to="2025-01-01T00:00:00"::DATETIME +| EVAL secs=DATE_DIFF("seconds", from, to), years=DATE_DIFF("year", from, to) +; + + from:date | to:date | secs:integer| years:integer +2024-01-01T00:00:00.000Z|2025-01-01T00:00:00.000Z|31622400 |1 +; + +evalDateDiffYearForDocs +required_capability: date_diff_year_calendarial + +// tag::evalDateDiffYearForDocs[] +ROW end_23="2023-12-31T23:59:59.999Z"::DATETIME, + start_24="2024-01-01T00:00:00.000Z"::DATETIME, + end_24="2024-12-31T23:59:59.999"::DATETIME +| EVAL end23_to_start24=DATE_DIFF("year", end_23, start_24) +| EVAL end23_to_end24=DATE_DIFF("year", end_23, end_24) +| EVAL start_to_end_24=DATE_DIFF("year", start_24, end_24) +// end::evalDateDiffYearForDocs[] +; + +// tag::evalDateDiffYearForDocs-result[] + end_23:date | start_24:date | end_24:date |end23_to_start24:integer|end23_to_end24:integer|start_to_end_24:integer +2023-12-31T23:59:59.999Z|2024-01-01T00:00:00.000Z|2024-12-31T23:59:59.999Z|0 |1 |0 +// end::evalDateDiffYearForDocs-result[] +; + evalDateParseWithSimpleDate row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b; @@ -480,6 +554,7 @@ emp_no:integer | year:long | month:long | day:long dateFormatLocale from employees | where emp_no == 10049 or emp_no == 10050 | sort emp_no | eval birth_month = date_format("MMMM", birth_date) | keep emp_no, birth_date, birth_month; +warningRegex:Date format \[MMMM\] contains textual field specifiers that could change in JDK 23 ignoreOrder:true emp_no:integer | birth_date:datetime | birth_month:keyword diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index e3160650521d9..597c349273eb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -314,7 +314,18 @@ public enum Cap { /** * QSTR function */ - QSTR_FUNCTION(true); + QSTR_FUNCTION(true), + + /** + * Don't optimize CASE IS NOT NULL function by not requiring the fields to be not null as well. + * https://github.com/elastic/elasticsearch/issues/112704 + */ + FIXED_WRONG_IS_NOT_NULL_CHECK_ON_CASE, + + /** + * Compute year differences in full calendar years. + */ + DATE_DIFF_YEAR_CALENDARIAL; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index 582785d023945..f9039417e48a6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -66,7 +66,7 @@ public class DateDiff extends EsqlScalarFunction { */ public enum Part implements DateTimeField { - YEAR((start, end) -> end.getYear() - start.getYear(), "years", "yyyy", "yy"), + YEAR((start, end) -> safeToInt(ChronoUnit.YEARS.between(start, end)), "years", "yyyy", "yy"), QUARTER((start, end) -> safeToInt(IsoFields.QUARTER_YEARS.between(start, end)), "quarters", "qq", "q"), MONTH((start, end) -> safeToInt(ChronoUnit.MONTHS.between(start, end)), "months", "mm", "m"), DAYOFYEAR((start, end) -> safeToInt(ChronoUnit.DAYS.between(start, end)), "dy", "y"), @@ -126,36 +126,44 @@ public static Part resolve(String dateTimeUnit) { } } - @FunctionInfo(returnType = "integer", description = """ - Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. - If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", detailedDescription = """ - [cols=\"^,^\",role=\"styled\"] - |=== - 2+h|Datetime difference units - - s|unit - s|abbreviations - - | year | years, yy, yyyy - | quarter | quarters, qq, q - | month | months, mm, m - | dayofyear | dy, y - | day | days, dd, d - | week | weeks, wk, ww - | weekday | weekdays, dw - | hour | hours, hh - | minute | minutes, mi, n - | second | seconds, ss, s - | millisecond | milliseconds, ms - | microsecond | microseconds, mcs - | nanosecond | nanoseconds, ns - |=== - - Note that while there is an overlap between the function's supported units and - {esql}'s supported time span literals, these sets are distinct and not - interchangeable. Similarly, the supported abbreviations are conveniently shared - with implementations of this function in other established products and not - necessarily common with the date-time nomenclature used by {es}.""", examples = @Example(file = "date", tag = "docsDateDiff")) + @FunctionInfo( + returnType = "integer", + description = """ + Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. + If `startTimestamp` is later than the `endTimestamp`, negative values are returned.""", + detailedDescription = """ + [cols=\"^,^\",role=\"styled\"] + |=== + 2+h|Datetime difference units + + s|unit + s|abbreviations + + | year | years, yy, yyyy + | quarter | quarters, qq, q + | month | months, mm, m + | dayofyear | dy, y + | day | days, dd, d + | week | weeks, wk, ww + | weekday | weekdays, dw + | hour | hours, hh + | minute | minutes, mi, n + | second | seconds, ss, s + | millisecond | milliseconds, ms + | microsecond | microseconds, mcs + | nanosecond | nanoseconds, ns + |=== + + Note that while there is an overlap between the function's supported units and + {esql}'s supported time span literals, these sets are distinct and not + interchangeable. Similarly, the supported abbreviations are conveniently shared + with implementations of this function in other established products and not + necessarily common with the date-time nomenclature used by {es}.""", + examples = { @Example(file = "date", tag = "docsDateDiff"), @Example(description = """ + When subtracting in calendar units - like year, month a.s.o. - only the fully elapsed units are counted. + To avoid this and obtain also remainders, simply switch to the next smaller unit and do the date math accordingly. + """, file = "date", tag = "evalDateDiffYearForDocs") } + ) public DateDiff( Source source, @Param(name = "unit", type = { "keyword", "text" }, description = "Time difference unit") Expression unit, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/InferIsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/InferIsNotNull.java index 81ae81bbba7b7..0e5bb74d1cdf9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/InferIsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/InferIsNotNull.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.rule.Rule; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -24,8 +25,7 @@ import static java.util.Collections.emptySet; /** - * Simplify IsNotNull targets by resolving the underlying expression to its root fields with unknown - * nullability. + * Simplify IsNotNull targets by resolving the underlying expression to its root fields. * e.g. * (x + 1) / 2 IS NOT NULL --> x IS NOT NULL AND (x+1) / 2 IS NOT NULL * SUBSTRING(x, 3) > 4 IS NOT NULL --> x IS NOT NULL AND SUBSTRING(x, 3) > 4 IS NOT NULL @@ -85,7 +85,7 @@ protected Set resolveExpressionAsRootAttributes(Expression exp, Attr private boolean doResolve(Expression exp, AttributeMap aliases, Set resolvedExpressions) { boolean changed = false; - // check if the expression can be skipped or is not nullabe + // check if the expression can be skipped if (skipExpression(exp)) { resolvedExpressions.add(exp); } else { @@ -106,6 +106,13 @@ private boolean doResolve(Expression exp, AttributeMap aliases, Set< } private static boolean skipExpression(Expression e) { - return e instanceof Coalesce; + // These two functions can have a complex set of expressions as arguments that can mess up the simplification we are trying to add. + // If there is a "case(f is null, null, ...) is not null" expression, + // assuming that "case(f is null.....) is not null AND f is not null" (what this rule is doing) is a wrong assumption because + // the "case" function will want both null "f" and not null "f". Doing it like this contradicts the condition inside case, so we + // must avoid these cases. + // We could be smarter and look inside "case" and "coalesce" to see if there is any comparison of fields with "null" but, + // the complexity is too high to warrant an attempt _now_. + return e instanceof Coalesce || e instanceof Case; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index 81f391d637317..4dbdfd1e56854 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -113,6 +113,34 @@ public static Iterable parameters() { ) ) ); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(1) + ); + })); + suppliers.add(new TestCaseSupplier("Date Diff In Year - 0", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> { + ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01.001Z"); + ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"), + new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"), + new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp") + ), + "DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]", + DataType.INTEGER, + equalTo(0) + ); + })); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index eba31cd1cf104..e556d43a471c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -58,8 +59,11 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.ONE; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_SEARCH_STATS; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.THREE; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TWO; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; @@ -81,10 +85,6 @@ public class LocalLogicalPlanOptimizerTests extends ESTestCase { private static LogicalPlanOptimizer logicalOptimizer; private static Map mapping; - private static final Literal ONE = L(1); - private static final Literal TWO = L(2); - private static final Literal THREE = L(3); - @BeforeClass public static void init() { parser = new EsqlParser(); @@ -386,38 +386,6 @@ public void testMissingFieldInFilterNoProjection() { ); } - public void testIsNotNullOnCoalesce() { - var plan = localPlan(""" - from test - | where coalesce(emp_no, salary) is not null - """); - - var limit = as(plan, Limit.class); - var filter = as(limit.child(), Filter.class); - var inn = as(filter.condition(), IsNotNull.class); - var coalesce = as(inn.children().get(0), Coalesce.class); - assertThat(Expressions.names(coalesce.children()), contains("emp_no", "salary")); - var source = as(filter.child(), EsRelation.class); - } - - public void testIsNotNullOnExpression() { - var plan = localPlan(""" - from test - | eval x = emp_no + 1 - | where x is not null - """); - - var limit = as(plan, Limit.class); - var filter = as(limit.child(), Filter.class); - var inn = as(filter.condition(), IsNotNull.class); - assertThat(Expressions.names(inn.children()), contains("x")); - var eval = as(filter.child(), Eval.class); - filter = as(eval.child(), Filter.class); - inn = as(filter.condition(), IsNotNull.class); - assertThat(Expressions.names(inn.children()), contains("emp_no")); - var source = as(filter.child(), EsRelation.class); - } - public void testSparseDocument() throws Exception { var query = """ from large @@ -516,6 +484,66 @@ public void testIsNotNullOnFunctionWithTwoFields() { assertEquals(expected, new InferIsNotNull().apply(f)); } + public void testIsNotNullOnCoalesce() { + var plan = localPlan(""" + from test + | where coalesce(emp_no, salary) is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + var coalesce = as(inn.children().get(0), Coalesce.class); + assertThat(Expressions.names(coalesce.children()), contains("emp_no", "salary")); + var source = as(filter.child(), EsRelation.class); + } + + public void testIsNotNullOnExpression() { + var plan = localPlan(""" + from test + | eval x = emp_no + 1 + | where x is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("x")); + var eval = as(filter.child(), Eval.class); + filter = as(eval.child(), Filter.class); + inn = as(filter.condition(), IsNotNull.class); + assertThat(Expressions.names(inn.children()), contains("emp_no")); + var source = as(filter.child(), EsRelation.class); + } + + public void testIsNotNullOnCase() { + var plan = localPlan(""" + from test + | where case(emp_no > 10000, "1", salary < 50000, "2", first_name) is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + var caseF = as(inn.children().get(0), Case.class); + assertThat(Expressions.names(caseF.children()), contains("emp_no > 10000", "\"1\"", "salary < 50000", "\"2\"", "first_name")); + var source = as(filter.child(), EsRelation.class); + } + + public void testIsNotNullOnCase_With_IS_NULL() { + var plan = localPlan(""" + from test + | where case(emp_no IS NULL, "1", salary IS NOT NULL, "2", first_name) is not null + """); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var inn = as(filter.condition(), IsNotNull.class); + var caseF = as(inn.children().get(0), Case.class); + assertThat(Expressions.names(caseF.children()), contains("emp_no IS NULL", "\"1\"", "salary IS NOT NULL", "\"2\"", "first_name")); + var source = as(filter.child(), EsRelation.class); + } + private IsNotNull isNotNull(Expression field) { return new IsNotNull(EMPTY, field); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index e181f77f2bcef..05edf6cdfb5a8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -29,6 +29,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' @@ -50,6 +51,7 @@ setup: - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" + - "Date format \\[MMMM\\] contains textual field specifiers that could change in JDK 23" esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml new file mode 100644 index 0000000000000..545b9ef1e0285 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/10_forbidden.yml @@ -0,0 +1,31 @@ +--- +"Test basic response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + info: {} + catch: unauthorized + + - match: + error.root_cause.0.type: security_exception + +--- +"Test bulk response with invalid credentials": + + - skip: + features: headers + + - do: + headers: + Authorization: "Basic dGVzdF91c2VyOndyb25nLXBhc3N3b3Jk" # invalid credentials + bulk: + body: | + {"index": {}} + {} + catch: unauthorized + - match: + error.root_cause.0.type: security_exception diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml index 1d370082c8e48..e1b297f1b5d78 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/snapshot/10_basic.yml @@ -90,3 +90,53 @@ setup: - match: {hits.total: 1 } - length: {hits.hits: 1 } - match: {hits.hits.0._id: "1" } + +--- +"Failed to snapshot indices with synthetic source": + - skip: + features: ["allowed_warnings"] + + - do: + indices.create: + index: test_synthetic + body: + mappings: + _source: + mode: synthetic + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_2 + wait_for_completion: true + body: | + { "indices": "test_synthetic" } + + - match: { snapshot.snapshot: test_snapshot_2 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 0 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason : "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} + + - do: + snapshot.create: + repository: test_repo_restore_1 + snapshot: test_snapshot_3 + wait_for_completion: true + body: | + { "indices": "test_*" } + + - match: { snapshot.snapshot: test_snapshot_3 } + - match: { snapshot.state : PARTIAL } + - match: { snapshot.shards.successful: 1 } + - match: { snapshot.shards.failed : 1 } + - match: { snapshot.failures.0.index: "test_synthetic" } + - match: { snapshot.failures.0.reason: "IllegalStateException[Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source]" } + - is_true: snapshot.version + - gt: { snapshot.version_id: 0} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java index 9212780d11fd3..5e6363b993ce4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java @@ -16,14 +16,12 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.ack.AckedRequest; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -69,42 +67,35 @@ protected void masterOperation( final boolean manuallyStopped = request.getCommand() == WatcherServiceRequest.Command.STOP; final String source = manuallyStopped ? "update_watcher_manually_stopped" : "update_watcher_manually_started"; - // TODO: make WatcherServiceRequest a real AckedRequest so that we have both a configurable timeout and master node timeout like - // we do elsewhere - submitUnbatchedTask(source, new AckedClusterStateUpdateTask(new AckedRequest() { - @Override - public TimeValue ackTimeout() { - return AcknowledgedRequest.DEFAULT_ACK_TIMEOUT; - } - - @Override - public TimeValue masterNodeTimeout() { - return request.masterNodeTimeout(); - } - }, listener) { - @Override - public ClusterState execute(ClusterState clusterState) { - XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); + // TODO: make WatcherServiceRequest a real AcknowledgedRequest so that we have both a configurable timeout and master node timeout + // like we do elsewhere + submitUnbatchedTask( + source, + new AckedClusterStateUpdateTask(request.masterNodeTimeout(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, listener) { + @Override + public ClusterState execute(ClusterState clusterState) { + XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); - WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); - WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); + WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped); + WatcherMetadata currentMetadata = clusterState.metadata().custom(WatcherMetadata.TYPE); - // adhere to the contract of returning the original state if nothing has changed - if (newWatcherMetadata.equals(currentMetadata)) { - return clusterState; - } else { - ClusterState.Builder builder = new ClusterState.Builder(clusterState); - builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); - return builder.build(); + // adhere to the contract of returning the original state if nothing has changed + if (newWatcherMetadata.equals(currentMetadata)) { + return clusterState; + } else { + ClusterState.Builder builder = new ClusterState.Builder(clusterState); + builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata)); + return builder.build(); + } } - } - @Override - public void onFailure(Exception e) { - logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); - listener.onFailure(e); + @Override + public void onFailure(Exception e) { + logger.error(() -> format("could not update watcher stopped status to [%s], source [%s]", manuallyStopped, source), e); + listener.onFailure(e); + } } - }); + ); } @Override