diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java index 09d32d79a508c..7ea9b41ca32f9 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java @@ -89,7 +89,6 @@ private Object[] getTargets(String projectPath) { return new String[] { "src/*/java/org/elasticsearch/action/admin/cluster/repositories/**/*.java", "src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java", - "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java", "src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java", "src/*/java/org/elasticsearch/index/IndexMode.java", "src/*/java/org/elasticsearch/index/IndexRouting.java", @@ -98,6 +97,10 @@ private Object[] getTargets(String projectPath) { "src/*/java/org/elasticsearch/repositories/**/*.java", "src/*/java/org/elasticsearch/search/aggregations/**/*.java", "src/*/java/org/elasticsearch/snapshots/**/*.java" }; + } else if (projectPath.equals(":test:framework")) { + return new String[] { + "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java", + }; } else { // Normally this isn"t necessary, but we have Java sources in // non-standard places @@ -203,7 +206,6 @@ private Object[] getTargets(String projectPath) { ":test:fixtures:geoip-fixture", ":test:fixtures:krb5kdc-fixture", ":test:fixtures:old-elasticsearch", - ":test:framework", ":test:logger-usage", ":x-pack:docs", ":x-pack:license-tools", diff --git a/build.gradle b/build.gradle index 1569cec42e9a5..60a2735f8cbe2 100644 --- a/build.gradle +++ b/build.gradle @@ -132,9 +132,9 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = true +boolean bwc_tests_enabled = false // place a PR link here when committing bwc changes: -String bwc_tests_disabled_issue = "" +String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/79385" /* * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java index 11e0c8f24f755..480278db72917 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java @@ -37,13 +37,13 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.transport.SniffConnectionStrategy; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.HashMap; @@ -316,7 +316,7 @@ public void testClusterHealthNotFoundIndex() throws IOException { assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED)); assertNoIndices(response); assertWarnings("The HTTP status code for a cluster health timeout will be changed from 408 to 200 in a " + - "future version. Set the [es.cluster_health.request_timeout_200] system property to [true] to suppress this message and " + + "future version. Set the [return_200_for_cluster_health_timeout] query parameter to [true] to suppress this message and " + "opt in to the future behaviour now."); } diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index e958a29d75bbc..92273b15e2e5e 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -97,6 +97,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] provided or better, i.e. `green` > `yellow` > `red`. By default, will not wait for any status. +`return_200_for_cluster_health_timeout`:: + (Optional, Boolean) A boolean value which controls whether to return HTTP 200 + status code instead of HTTP 408 in case of a cluster health timeout from + the server side. Defaults to false. + [[cluster-health-api-response-body]] ==== {api-response-body-title} diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index a5679bfe570f8..99b015e4f717a 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -507,7 +507,7 @@ include::{es-repo-dir}/tab-widgets/quick-start-cleanup-widget.asciidoc[] * Use {fleet} and {agent} to collect logs and metrics directly from your data sources and send them to {es}. See the -{fleet-guide}/fleet-quick-start.html[{fleet} quick start guide]. +{observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}]. * Use {kib} to explore, visualize, and manage your {es} data. See the {kibana-ref}/get-started.html[{kib} quick start guide]. diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index b0d6859a051f3..5280dc160cdd6 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -432,8 +432,7 @@ If you run {agent} standalone, you can apply pipelines using an <> or <> index setting. Alternatively, you can specify the `pipeline` policy setting in your `elastic-agent.yml` -configuration. See {fleet-guide}/run-elastic-agent-standalone.html[Run {agent} -standalone]. +configuration. See {fleet-guide}/install-standalone-elastic-agent.html[Install standalone {agent}s]. [discrete] [[access-source-fields]] diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java index 8c23a71965e73..89ccb670c5c3a 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/InstantiatingObjectParser.java @@ -23,8 +23,12 @@ *

* The main differences being that it is using Builder to construct the parser and takes a class of the target object instead of the object * builder. The target object must have exactly one constructor with the number and order of arguments matching the number of order of - * declared fields. If there are more then 2 constructors with the same number of arguments, one of them needs to be marked with + * declared fields. If there are more than 2 constructors with the same number of arguments, one of them needs to be marked with * {@linkplain ParserConstructor} annotation. + * + * It is also possible for the constructor to accept Context as the first parameter, in this case as in the case with multiple constructors + * it is required for the constructor to be marked with {@linkplain ParserConstructor} annotation. + * * 

{@code
  *   public static class Thing{
  *       public Thing(String animal, String vegetable, int mineral) {
@@ -37,14 +41,35 @@
  *
  *   }
  *
- *   private static final InstantiatingObjectParser PARSER = new InstantiatingObjectParser<>("thing", Thing.class);
+ *   private static final InstantiatingObjectParser PARSER;
+ *   static {
+ *       InstantiatingObjectParser.Builder parser =
+ *           InstantiatingObjectParser,builder<>("thing", true, Thing.class);
+ *       parser.declareString(constructorArg(), new ParseField("animal"));
+ *       parser.declareString(constructorArg(), new ParseField("vegetable"));
+ *       parser.declareInt(optionalConstructorArg(), new ParseField("mineral"));
+ *       parser.declareInt(Thing::setFruit, new ParseField("fruit"));
+ *       parser.declareInt(Thing::setBug, new ParseField("bug"));
+ *       PARSER = parser.build()
+ *   }
+ * }
+ * 
{@code
+ *
+ *   public static class AnotherThing {
+ *       @ParserConstructor
+ *       public AnotherThing(SomeContext continent, String animal, String vegetable, int mineral) {
+ *           ....
+ *       }
+ *   }
+ *
+ *   private static final InstantiatingObjectParser PARSER;
  *   static {
- *       PARSER.declareString(constructorArg(), new ParseField("animal"));
- *       PARSER.declareString(constructorArg(), new ParseField("vegetable"));
- *       PARSER.declareInt(optionalConstructorArg(), new ParseField("mineral"));
- *       PARSER.declareInt(Thing::setFruit, new ParseField("fruit"));
- *       PARSER.declareInt(Thing::setBug, new ParseField("bug"));
- *       PARSER.finalizeFields()
+ *       InstantiatingObjectParser.Builder parser =
+ *           InstantiatingObjectParser,builder<>("thing", true, AnotherThing.class);
+ *       parser.declareString(constructorArg(), new ParseField("animal"));
+ *       parser.declareString(constructorArg(), new ParseField("vegetable"));
+ *       parser.declareInt(optionalConstructorArg(), new ParseField("mineral"));
+ *       PARSER = parser.build()
  *   }
  * }
*/ @@ -72,7 +97,7 @@ public Builder(String name, Class valueClass) { } public Builder(String name, boolean ignoreUnknownFields, Class valueClass) { - this.constructingObjectParser = new ConstructingObjectParser<>(name, ignoreUnknownFields, this::build); + this.constructingObjectParser = new ConstructingObjectParser<>(name, ignoreUnknownFields, this::buildInstance); this.valueClass = valueClass; } @@ -87,9 +112,15 @@ public InstantiatingObjectParser build() { throw new IllegalArgumentException("More then one public constructor with @ParserConstructor annotation exist in " + "the class " + valueClass.getName()); } - if (c.getParameterCount() != neededArguments) { - throw new IllegalArgumentException("Annotated constructor doesn't have " + neededArguments + - " arguments in the class " + valueClass.getName()); + if (c.getParameterCount() < neededArguments || c.getParameterCount() > neededArguments + 1) { + throw new IllegalArgumentException( + "Annotated constructor doesn't have " + + neededArguments + + " or " + + (neededArguments + 1) + + " arguments in the class " + + valueClass.getName() + ); } constructor = c; } @@ -154,13 +185,20 @@ public void declareExclusiveFieldSet(String... exclusiveSet) { constructingObjectParser.declareExclusiveFieldSet(exclusiveSet); } - private Value build(Object[] args) { + private Value buildInstance(Object[] args, Context context) { if (constructor == null) { throw new IllegalArgumentException("InstantiatingObjectParser for type " + valueClass.getName() + " has to be finalized " + "before the first use"); } try { - return constructor.newInstance(args); + if (constructor.getParameterCount() != args.length) { + Object[] newArgs = new Object[args.length + 1]; + System.arraycopy(args, 0, newArgs, 1, args.length); + newArgs[0] = context; + return constructor.newInstance(newArgs); + } else { + return constructor.newInstance(args); + } } catch (Exception ex) { throw new IllegalArgumentException("Cannot instantiate an object of " + valueClass.getName(), ex); } diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java index db155c2334851..34f02b373582e 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/InstantiatingObjectParserTests.java @@ -8,11 +8,8 @@ package org.elasticsearch.xcontent; -import org.elasticsearch.xcontent.InstantiatingObjectParser; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ParserConstructor; -import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.Objects; @@ -217,8 +214,10 @@ public void testAnnotationWrongArgumentNumber() { InstantiatingObjectParser.Builder builder = InstantiatingObjectParser.builder("foo", Annotations.class); builder.declareInt(constructorArg(), new ParseField("a")); builder.declareString(constructorArg(), new ParseField("b")); + builder.declareInt(constructorArg(), new ParseField("c")); + builder.declareString(constructorArg(), new ParseField("d")); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 2 arguments in the class")); + assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 4 or 5 arguments in the class")); } public void testDoubleDeclarationThrowsException() throws IOException { @@ -240,4 +239,80 @@ class DoubleFieldDeclaration { assertThat(exception, instanceOf(IllegalArgumentException.class)); assertThat(exception.getMessage(), startsWith("Parser already registered for name=[name]")); } + + public static class ContextArgument { + final String context; + final int a; + final String b; + final long c; + + public ContextArgument() { + this(1, "2", 3); + } + + public ContextArgument(int a, String b) { + this(a, b, -1); + } + + + public ContextArgument(int a, String b, long c) { + this(null, a, b, c); + } + + public ContextArgument(String context, int a, String b, long c) { + this.context = context; + this.a = a; + this.b = b; + this.c = c; + } + + @ParserConstructor + public ContextArgument(String context, int a, String b, String c) { + this.context = context; + this.a = a; + this.b = b; + this.c = Long.parseLong(c); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContextArgument that = (ContextArgument) o; + return a == that.a && + c == that.c && + Objects.equals(b, that.b); + } + + @Override + public int hashCode() { + return Objects.hash(a, b, c); + } + } + + public void testContextAsArgument() throws IOException { + InstantiatingObjectParser.Builder builder = InstantiatingObjectParser.builder( + "foo", + ContextArgument.class + ); + builder.declareInt(constructorArg(), new ParseField("a")); + builder.declareString(constructorArg(), new ParseField("b")); + builder.declareString(constructorArg(), new ParseField("c")); + InstantiatingObjectParser parser = builder.build(); + try (XContentParser contentParser = createParser(JsonXContent.jsonXContent, "{\"a\": 5, \"b\":\"6\", \"c\": \"7\"}")) { + assertThat(parser.parse(contentParser, "context"), equalTo(new ContextArgument("context", 5, "6", 7))); + } + } + + public void testContextAsArgumentWrongArgumentNumber() { + InstantiatingObjectParser.Builder builder = InstantiatingObjectParser.builder( + "foo", + ContextArgument.class + ); + builder.declareInt(constructorArg(), new ParseField("a")); + builder.declareString(constructorArg(), new ParseField("b")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 2 or 3 arguments in the class")); + } + } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java similarity index 98% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 87d4d92c7a910..af7e9aae149d8 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -6,10 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.support.AbstractFilteringTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContent; @@ -17,8 +18,6 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.common.xcontent.support.AbstractFilteringTestCase; -import org.elasticsearch.xcontent.support.filtering.FilterPath; import java.io.IOException; import java.util.Arrays; @@ -142,6 +141,8 @@ static void assertXContentBuilderAsBytes(final XContentBuilder expected, final X assertThat(jsonParser.numberType(), equalTo(testParser.numberType())); assertThat(jsonParser.numberValue(), equalTo(testParser.numberValue())); break; + default: + break; } } } catch (Exception e) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java index ce4c5d005c759..5b2dce8e10106 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/CborXContentFilteringTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java index 3d73c8717e7ef..ad669a3e61b5d 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -6,14 +6,14 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate; + import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.support.filtering.FilterPathBasedFilter; import java.util.Collections; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java index 2046772e0afcf..c8a65e90a4c3a 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java @@ -6,10 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.support.filtering.FilterPath; import java.util.Arrays; import java.util.LinkedHashSet; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java index fffdbb2ad8818..5a27954754d43 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/JsonXContentFilteringTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java index 7c54668d17192..13efcc0738949 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/SmileFilteringGeneratorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java rename to libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java index 00769671707a2..ada8b696c5d64 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/YamlFilteringGeneratorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.common.xcontent.support.filtering; +package org.elasticsearch.xcontent.support.filtering; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_versioned_update.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_versioned_update.yml index 780f33be52dc0..68fa8d6c86014 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_versioned_update.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/290_versioned_update.yml @@ -1,8 +1,8 @@ --- "Test pipeline versioned updates": - skip: - version: " - 7.99.99" - reason: "re-enable in 7.16+ when backported" + version: " - 7.15.99" + reason: "added versioned updates in 7.16.0" - do: ingest.put_pipeline: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json index 91712bbbded29..7d33fdd52ab81 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json @@ -102,6 +102,10 @@ "red" ], "description":"Wait until cluster is in a specific state" + }, + "return_200_for_cluster_health_timeout":{ + "type":"boolean", + "description":"Whether to return HTTP 200 instead of 408 in case of a cluster health timeout from the server side" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/knn_search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/knn_search.json new file mode 100644 index 0000000000000..b55f35ccab4fe --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/knn_search.json @@ -0,0 +1,40 @@ +{ + "knn_search":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-search.html", + "description":"Performs a kNN search." + }, + "stability":"experimental", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/{index}/_knn_search", + "methods":[ + "GET", + "POST" + ], + "parts":{ + "index":{ + "type":"list", + "description":"A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + } + } + } + ] + }, + "params": { + "routing":{ + "type":"list", + "description":"A comma-separated list of specific routing values" + } + }, + "body":{ + "description":"The search definition" + } + } +} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml index 66a7cb2b48dbd..e5a4db4dbfd9f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml @@ -35,3 +35,25 @@ - match: { initializing_shards: 0 } - match: { unassigned_shards: 0 } - gte: { number_of_pending_tasks: 0 } + +--- +"cluster health request timeout with 200 response code": + - skip: + version: " - 7.99.99" + reason: "return_200_for_cluster_health_timeout exists only in 8.0.0; re-enable in 7.16+ when back-ported" + - do: + cluster.health: + timeout: 1ms + wait_for_active_shards: 5 + return_200_for_cluster_health_timeout: true + + - is_true: cluster_name + - is_true: timed_out + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - match: { active_primary_shards: 0 } + - match: { active_shards: 0 } + - match: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - gte: { number_of_pending_tasks: 0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml index 05937b73324bd..cd971882316d9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.put_settings/10_basic.yml @@ -1,7 +1,7 @@ --- "Test put and reset transient settings": - skip: - version: " - 7.99.99" + version: " - 7.15.99" reason: "transient settings deprecation" features: "warnings" diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 8842856aa3fa6..d4aa7f1177bd5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -35,6 +35,8 @@ public class ClusterHealthRequest extends MasterNodeReadRequest INDEX_PARSER = (XContentParser parser, Void context, String index) -> ClusterIndexHealth.innerFromXContent(parser, index); - private static final String ES_CLUSTER_HEALTH_REQUEST_TIMEOUT_200_KEY = "es.cluster_health.request_timeout_200"; + static final String ES_CLUSTER_HEALTH_REQUEST_TIMEOUT_200_KEY = "return_200_for_cluster_health_timeout"; static final String CLUSTER_HEALTH_REQUEST_TIMEOUT_DEPRECATION_MSG = "The HTTP status code for a cluster health timeout " + "will be changed from 408 to 200 in a future version. Set the [" + ES_CLUSTER_HEALTH_REQUEST_TIMEOUT_200_KEY + "] " + - "system property to [true] to suppress this message and opt in to the future behaviour now."; + "query parameter to [true] to suppress this message and opt in to the future behaviour now."; static { // ClusterStateHealth fields @@ -137,15 +138,7 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo private boolean timedOut = false; private ClusterStateHealth clusterStateHealth; private ClusterHealthStatus clusterHealthStatus; - private boolean esClusterHealthRequestTimeout200 = readEsClusterHealthRequestTimeout200FromProperty(); - - public ClusterHealthResponse() { - } - - /** For the testing of opting in for the 200 status code without setting a system property */ - ClusterHealthResponse(boolean esClusterHealthRequestTimeout200) { - this.esClusterHealthRequestTimeout200 = esClusterHealthRequestTimeout200; - } + private boolean return200ForClusterHealthTimeout; public ClusterHealthResponse(StreamInput in) throws IOException { super(in); @@ -157,15 +150,21 @@ public ClusterHealthResponse(StreamInput in) throws IOException { numberOfInFlightFetch = in.readInt(); delayedUnassignedShards= in.readInt(); taskMaxWaitingTime = in.readTimeValue(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + return200ForClusterHealthTimeout = in.readBoolean(); + } } /** needed for plugins BWC */ - public ClusterHealthResponse(String clusterName, String[] concreteIndices, ClusterState clusterState) { - this(clusterName, concreteIndices, clusterState, -1, -1, -1, TimeValue.timeValueHours(0)); + public ClusterHealthResponse(String clusterName, String[] concreteIndices, ClusterState clusterState, + boolean return200ForServerTimeout) { + this(clusterName, concreteIndices, clusterState, -1, -1, -1, TimeValue.timeValueHours(0), + return200ForServerTimeout); } public ClusterHealthResponse(String clusterName, String[] concreteIndices, ClusterState clusterState, int numberOfPendingTasks, - int numberOfInFlightFetch, int delayedUnassignedShards, TimeValue taskMaxWaitingTime) { + int numberOfInFlightFetch, int delayedUnassignedShards, TimeValue taskMaxWaitingTime, + boolean return200ForServerTimeout) { this.clusterName = clusterName; this.numberOfPendingTasks = numberOfPendingTasks; this.numberOfInFlightFetch = numberOfInFlightFetch; @@ -173,6 +172,7 @@ public ClusterHealthResponse(String clusterName, String[] concreteIndices, Clust this.taskMaxWaitingTime = taskMaxWaitingTime; this.clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices); this.clusterHealthStatus = clusterStateHealth.getStatus(); + this.return200ForClusterHealthTimeout = return200ForServerTimeout; } /** @@ -304,6 +304,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(numberOfInFlightFetch); out.writeInt(delayedUnassignedShards); out.writeTimeValue(taskMaxWaitingTime); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(return200ForClusterHealthTimeout); + } else if (return200ForClusterHealthTimeout) { + throw new IllegalArgumentException("Can't fix response code in a cluster involving nodes with version " + out.getVersion()); + } } @Override @@ -316,7 +321,7 @@ public RestStatus status() { if (isTimedOut() == false) { return RestStatus.OK; } - if (esClusterHealthRequestTimeout200) { + if (return200ForClusterHealthTimeout) { return RestStatus.OK; } else { deprecationLogger.compatibleCritical("cluster_health_request_timeout", CLUSTER_HEALTH_REQUEST_TIMEOUT_DEPRECATION_MSG); @@ -381,17 +386,4 @@ public int hashCode() { return Objects.hash(clusterName, numberOfPendingTasks, numberOfInFlightFetch, delayedUnassignedShards, taskMaxWaitingTime, timedOut, clusterStateHealth, clusterHealthStatus); } - - private static boolean readEsClusterHealthRequestTimeout200FromProperty() { - String property = System.getProperty(ES_CLUSTER_HEALTH_REQUEST_TIMEOUT_200_KEY); - if (property == null) { - return false; - } - if (Boolean.parseBoolean(property)) { - return true; - } else { - throw new IllegalArgumentException(ES_CLUSTER_HEALTH_REQUEST_TIMEOUT_200_KEY + " can only be unset or [true] but was [" - + property + "]"); - } - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java index 83d4469e3b19d..ee261c253a2e6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -30,8 +30,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; @@ -225,7 +225,8 @@ private enum TimeoutState { private ClusterHealthResponse getResponse(final ClusterHealthRequest request, ClusterState clusterState, final int waitFor, TimeoutState timeoutState) { - ClusterHealthResponse response = clusterHealth(request, clusterState, clusterService.getMasterService().numberOfPendingTasks(), + ClusterHealthResponse response = clusterHealth(request, clusterState, + clusterService.getMasterService().numberOfPendingTasks(), allocationService.getNumberOfInFlightFetches(), clusterService.getMasterService().getMaxTaskWaitTime()); int readyCounter = prepareResponse(request, response, clusterState, indexNameExpressionResolver); boolean valid = (readyCounter == waitFor); @@ -324,8 +325,8 @@ static int prepareResponse(final ClusterHealthRequest request, final ClusterHeal } - private ClusterHealthResponse clusterHealth(ClusterHealthRequest request, ClusterState clusterState, int numberOfPendingTasks, - int numberOfInFlightFetch, TimeValue pendingTaskTimeInQueue) { + private ClusterHealthResponse clusterHealth(ClusterHealthRequest request, ClusterState clusterState, + int numberOfPendingTasks, int numberOfInFlightFetch, TimeValue pendingTaskTimeInQueue) { if (logger.isTraceEnabled()) { logger.trace("Calculating health based on state version [{}]", clusterState.version()); } @@ -337,12 +338,13 @@ private ClusterHealthResponse clusterHealth(ClusterHealthRequest request, Cluste // one of the specified indices is not there - treat it as RED. ClusterHealthResponse response = new ClusterHealthResponse(clusterState.getClusterName().value(), Strings.EMPTY_ARRAY, clusterState, numberOfPendingTasks, numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), - pendingTaskTimeInQueue); + pendingTaskTimeInQueue, request.doesReturn200ForClusterHealthTimeout()); response.setStatus(ClusterHealthStatus.RED); return response; } - return new ClusterHealthResponse(clusterState.getClusterName().value(), concreteIndices, clusterState, numberOfPendingTasks, - numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue); + return new ClusterHealthResponse(clusterState.getClusterName().value(), concreteIndices, clusterState, + numberOfPendingTasks, numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue, + request.doesReturn200ForClusterHealthTimeout()); } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index f0a7532dc80ac..13a9d28dadf4e 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -10,12 +10,14 @@ import org.elasticsearch.Version; import org.elasticsearch.index.mapper.TimeSeriesParams; +import org.elasticsearch.xcontent.InstantiatingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParserConstructor; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -159,6 +161,59 @@ public FieldCapabilities(String name, String type, } + /** + * Constructor for a set of indices used by parser + * @param name The name of the field + * @param type The type associated with the field. + * @param isMetadataField Whether this field is a metadata field. + * @param isSearchable Whether this field is indexed for search. + * @param isAggregatable Whether this field can be aggregated on. + * @param isDimension Whether this field can be used as dimension + * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields + * @param indices The list of indices where this field name is defined as {@code type}, + * or null if all indices have the same {@code type} for the field. + * @param nonSearchableIndices The list of indices where this field is not searchable, + * or null if the field is searchable in all indices. + * @param nonAggregatableIndices The list of indices where this field is not aggregatable, + * or null if the field is aggregatable in all indices. + * @param nonDimensionIndices The list of indices where this field is not a dimension + * @param metricConflictsIndices The list of indices where this field is has different metric types or not mark as a metric + * @param meta Merged metadata across indices. + */ + @SuppressWarnings("unused") + @ParserConstructor + public FieldCapabilities( + String name, + String type, + Boolean isMetadataField, + boolean isSearchable, + boolean isAggregatable, + Boolean isDimension, + String metricType, + List indices, + List nonSearchableIndices, + List nonAggregatableIndices, + List nonDimensionIndices, + List metricConflictsIndices, + Map> meta + ) { + this( + name, + type, + isMetadataField == null ? false : isMetadataField, + isSearchable, + isAggregatable, + isDimension == null ? false : isDimension, + metricType != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, metricType) : null, + indices != null ? indices.toArray(new String[0]) : null, + nonSearchableIndices != null ? nonSearchableIndices.toArray(new String[0]) : null, + nonAggregatableIndices != null ? nonAggregatableIndices.toArray(new String[0]) : null, + nonDimensionIndices != null ? nonDimensionIndices.toArray(new String[0]) : null, + metricConflictsIndices != null ? metricConflictsIndices.toArray(new String[0]) : null, + meta != null ? meta : Collections.emptyMap() + ); + } + FieldCapabilities(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); @@ -254,43 +309,31 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser) } @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "field_capabilities", - true, - (a, name) -> new FieldCapabilities( - name, - (String) a[0], - a[3] == null ? false : (boolean) a[3], - (boolean) a[1], - (boolean) a[2], - a[4] == null ? false : (boolean) a[4], - a[5] != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, (String) a[5]) : null, - a[6] != null ? ((List) a[6]).toArray(new String[0]) : null, - a[7] != null ? ((List) a[7]).toArray(new String[0]) : null, - a[8] != null ? ((List) a[8]).toArray(new String[0]) : null, - a[9] != null ? ((List) a[9]).toArray(new String[0]) : null, - a[10] != null ? ((List) a[10]).toArray(new String[0]) : null, - a[11] != null ? ((Map>) a[11]) : Collections.emptyMap() - ) - ); + private static final InstantiatingObjectParser PARSER; static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); // 0 - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); // 1 - PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); // 2 - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); // 3 - PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_DIMENSION_FIELD); // 4 - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_METRIC_FIELD); // 5 - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); // 6 - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); // 7 - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); // 8 - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_DIMENSION_INDICES_FIELD); // 9 - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), METRIC_CONFLICTS_INDICES_FIELD); // 10 - PARSER.declareObject( + InstantiatingObjectParser.Builder parser = InstantiatingObjectParser.builder( + "field_capabilities", + true, + FieldCapabilities.class + ); + parser.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); + parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), IS_METADATA_FIELD); + parser.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); + parser.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); + parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_DIMENSION_FIELD); + parser.declareString(ConstructingObjectParser.optionalConstructorArg(), TIME_SERIES_METRIC_FIELD); + parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); + parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); + parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); + parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_DIMENSION_INDICES_FIELD); + parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), METRIC_CONFLICTS_INDICES_FIELD); + parser.declareObject( ConstructingObjectParser.optionalConstructorArg(), - (parser, context) -> parser.map(HashMap::new, p -> Set.copyOf(p.list())), + (p, context) -> p.map(HashMap::new, v -> Set.copyOf(v.list())), META_FIELD - ); // 11 + ); + PARSER = parser.build(); } /** diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index b5cf692723dd7..cfdb932199b88 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -71,7 +71,7 @@ public TransportFieldCapabilitiesAction(TransportService transportService, this.fieldCapabilitiesFetcher = new FieldCapabilitiesFetcher(indicesService); final Set metadataFields = indicesService.getAllMetadataFields(); this.metadataFieldPred = metadataFields::contains; - transportService.registerRequestHandler(ACTION_NODE_NAME, ThreadPool.Names.MANAGEMENT, + transportService.registerRequestHandler(ACTION_NODE_NAME, ThreadPool.Names.SEARCH_COORDINATION, FieldCapabilitiesNodeRequest::new, new NodeTransportHandler()); } @@ -111,7 +111,7 @@ protected void doExecute(Task task, FieldCapabilitiesRequest request, final Acti localIndices, nowInMillis, concreteIndices, - threadPool.executor(ThreadPool.Names.MANAGEMENT), + threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION), indexResponse -> indexResponses.putIfAbsent(indexResponse.getIndexName(), indexResponse), indexFailures::collect, countDown @@ -163,7 +163,7 @@ private Runnable createResponseMerger(FieldCapabilitiesRequest request, if (request.isMergeResults()) { // fork off to the management pool for merging the responses as the operation can run for longer than is acceptable // on a transport thread in case of large numbers of indices and/or fields - threadPool.executor(ThreadPool.Names.MANAGEMENT).submit( + threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION).submit( ActionRunnable.supply( listener, () -> merge(indexResponses, request.includeUnmapped(), new ArrayList<>(failures))) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 918bbf426b2b8..96fd61c9f7010 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -48,7 +48,7 @@ public PutPipelineRequest(StreamInput in) throws IOException { id = in.readString(); source = in.readBytesReference(); xContentType = in.readEnum(XContentType.class); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_16_0)) { version = in.readOptionalInt(); } else { version = null; @@ -86,7 +86,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(source); XContentHelper.writeTo(out, xContentType); - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_16_0)) { out.writeOptionalInt(version); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java index 131112a0ad29f..ca2f4653f85df 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java @@ -81,6 +81,9 @@ public static ClusterHealthRequest fromRequest(final RestRequest request) { if (request.param("wait_for_events") != null) { clusterHealthRequest.waitForEvents(Priority.valueOf(request.param("wait_for_events").toUpperCase(Locale.ROOT))); } + clusterHealthRequest.return200ForClusterHealthTimeout(request.paramAsBoolean( + "return_200_for_cluster_health_timeout", + clusterHealthRequest.doesReturn200ForClusterHealthTimeout())); return clusterHealthRequest; } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index af67aeb1fd0ae..46ac90b42a840 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -16,13 +16,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentFragment; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; @@ -49,6 +43,12 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java index 514d59d45c0ef..8608d9f3a6886 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequestTests.java @@ -43,6 +43,7 @@ public void testSerialize() throws Exception { assertThat(cloneRequest.waitForEvents(), equalTo(originalRequest.waitForEvents())); assertIndicesEquals(cloneRequest.indices(), originalRequest.indices()); assertThat(cloneRequest.indicesOptions(), equalTo(originalRequest.indicesOptions())); + assertThat(cloneRequest.doesReturn200ForClusterHealthTimeout(), equalTo(originalRequest.doesReturn200ForClusterHealthTimeout())); } public void testRequestReturnsHiddenIndicesByDefault() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index 01ad7731d0f76..fd3a63dd290a1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -20,10 +20,10 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; import org.hamcrest.Matchers; import java.io.IOException; @@ -43,7 +43,7 @@ public class ClusterHealthResponsesTests extends AbstractSerializingTestCase void sendResponse(TransportResponseHandler handler, TransportResponse resp) { - threadPool.executor(ThreadPool.Names.MANAGEMENT).submit(new AbstractRunnable() { + threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION).submit(new AbstractRunnable() { @Override public void onFailure(Exception e) { throw new AssertionError(e); @@ -765,20 +764,6 @@ protected void doRun() { } }); } - - void sendFailure(TransportResponseHandler handler, Exception e) { - threadPool.executor(ThreadPool.Names.MANAGEMENT).submit(new AbstractRunnable() { - @Override - public void onFailure(Exception e) { - throw new AssertionError(e); - } - - @Override - protected void doRun() { - handler.handleException(new TransportException(e)); - } - }); - } } static FieldCapabilitiesRequest randomFieldCapRequest(boolean withFilter) { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index 92e9d6ac5aed4..afe24a65df677 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LogEvent; +import org.apache.lucene.util.Constants; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; @@ -1847,13 +1848,17 @@ public void testLogsMessagesIfPublicationDelayed() throws IllegalAccessException "node [" + brokenNode + "] is lagging at cluster state version [*], " + "although publication of cluster state version [*] completed [*] ago")); - mockLogAppender.addExpectation(new MockLogAppender.SeenEventExpectation( - "hot threads from lagging node", - LagDetector.class.getCanonicalName(), - Level.DEBUG, - "hot threads from node [" + - brokenNode.getLocalNode().descriptionWithoutAttributes() + - "] lagging at version [*] despite commit of cluster state version [*]:\nHot threads at*")); + if (Constants.WINDOWS == false) { + // log messages containing control characters are hidden from the log assertions framework, and this includes the + // `\r` that Windows uses in its line endings, so we only see this message on systems with `\n` line endings: + mockLogAppender.addExpectation(new MockLogAppender.SeenEventExpectation( + "hot threads from lagging node", + LagDetector.class.getCanonicalName(), + Level.DEBUG, + "hot threads from node [" + + brokenNode.getLocalNode().descriptionWithoutAttributes() + + "] lagging at version [*] despite commit of cluster state version [*]:\nHot threads at*")); + } // drop the publication messages to one node, but then restore connectivity so it remains in the cluster and does not fail // health checks diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index c5f6dff52be66..14b733bee3952 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -316,12 +316,7 @@ public void testPrefixedNamesFilteringTest() { public void testNestedFiltering() { Map map = new HashMap<>(); map.put("field", "value"); - map.put("array", Arrays.asList(1, new HashMap() { - { - put("nested", 2); - put("nested_2", 3); - } - })); + map.put("array", Arrays.asList(1, Map.of("nested", 2, "nested_2", 3))); Map filteredMap = XContentMapValues.filter(map, new String[] { "array.nested" }, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); @@ -336,12 +331,7 @@ public void testNestedFiltering() { map.clear(); map.put("field", "value"); - map.put("obj", new HashMap() { - { - put("field", "value"); - put("field2", "value2"); - } - }); + map.put("obj", Map.of("field", "value", "field2", "value2")); filteredMap = XContentMapValues.filter(map, new String[] { "obj.field" }, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); assertThat(((Map) filteredMap.get("obj")).size(), equalTo(1)); @@ -359,18 +349,8 @@ public void testNestedFiltering() { public void testCompleteObjectFiltering() { Map map = new HashMap<>(); map.put("field", "value"); - map.put("obj", new HashMap() { - { - put("field", "value"); - put("field2", "value2"); - } - }); - map.put("array", Arrays.asList(1, new HashMap() { - { - put("field", "value"); - put("field2", "value2"); - } - })); + map.put("obj", Map.of("field", "value", "field2", "value2")); + map.put("array", Arrays.asList(1, Map.of("field", "value", "field2", "value2"))); Map filteredMap = XContentMapValues.filter(map, new String[] { "obj" }, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); @@ -401,18 +381,8 @@ public void testCompleteObjectFiltering() { public void testFilterIncludesUsingStarPrefix() { Map map = new HashMap<>(); map.put("field", "value"); - map.put("obj", new HashMap() { - { - put("field", "value"); - put("field2", "value2"); - } - }); - map.put("n_obj", new HashMap() { - { - put("n_field", "value"); - put("n_field2", "value2"); - } - }); + map.put("obj", Map.of("field", "value", "field2", "value2")); + map.put("n_obj", Map.of("n_field", "value", "n_field2", "value2")); Map filteredMap = XContentMapValues.filter(map, new String[] { "*.field2" }, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); @@ -546,6 +516,11 @@ public void testDotsInFieldNames() { assertEquals(expected, filtered); } + /** + * Tests that we can extract paths containing non-ascii characters. + * See {@link AbstractFilteringTestCase#testFilterSupplementaryCharactersInPaths()} + * for a similar test but for XContent. + */ public void testSupplementaryCharactersInPaths() { Map map = new HashMap<>(); map.put("搜索", 2); @@ -555,6 +530,11 @@ public void testSupplementaryCharactersInPaths() { assertEquals(Collections.singletonMap("指数", 3), XContentMapValues.filter(map, new String[0], new String[] { "搜索" })); } + /** + * Tests that we can extract paths which share a prefix with other paths. + * See {@link AbstractFilteringTestCase#testFilterSharedPrefixes()} + * for a similar test but for XContent. + */ public void testSharedPrefixes() { Map map = new HashMap<>(); map.put("foobar", 2); @@ -633,6 +613,11 @@ public void testEmptyObjectsSubFieldsInclusion() { } } + /** + * Tests that we can extract paths which have another path as a prefix. + * See {@link AbstractFilteringTestCase#testFilterPrefix()} + * for a similar test but for XContent. + */ public void testPrefix() { Map map = new HashMap<>(); map.put("photos", Arrays.asList(new String[] { "foo", "bar" })); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthActionTests.java index 5e9e5662d6e04..e42c49f2f93b2 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthActionTests.java @@ -51,6 +51,8 @@ public void testFromRequest() { params.put("wait_for_active_shards", String.valueOf(waitForActiveShards)); params.put("wait_for_nodes", waitForNodes); params.put("wait_for_events", waitForEvents.name()); + boolean requestTimeout200 = randomBoolean(); + params.put("return_200_for_cluster_health_timeout", String.valueOf(requestTimeout200)); FakeRestRequest restRequest = buildRestRequest(params); ClusterHealthRequest clusterHealthRequest = RestClusterHealthAction.fromRequest(restRequest); @@ -65,7 +67,7 @@ public void testFromRequest() { assertThat(clusterHealthRequest.waitForActiveShards(), equalTo(ActiveShardCount.parseString(String.valueOf(waitForActiveShards)))); assertThat(clusterHealthRequest.waitForNodes(), equalTo(waitForNodes)); assertThat(clusterHealthRequest.waitForEvents(), equalTo(waitForEvents)); - + assertThat(clusterHealthRequest.doesReturn200ForClusterHealthTimeout(), equalTo(requestTimeout200)); } private FakeRestRequest buildRestRequest(Map params) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java similarity index 92% rename from server/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java rename to test/framework/src/main/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java index fbb5a8c84372b..c5bb14133042e 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.PathUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.FilterXContentParser; @@ -21,15 +20,18 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.Set; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; import static java.util.stream.Collectors.toSet; +import static org.hamcrest.Matchers.notNullValue; /** * Tests for {@link XContent} filtering. @@ -46,17 +48,16 @@ protected interface Builder extends CheckedFunction { - try ( - XContentParser parser = XContentType.JSON.xContent() - .createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - AbstractFilteringTestCase.class.getResourceAsStream(file) - ) - ) { - // copyCurrentStructure does not property handle filters when it is passed a json parser. So we hide it. - return builder.copyCurrentStructure(new FilterXContentParser(parser) { - }); + try (InputStream stream = AbstractFilteringTestCase.class.getResourceAsStream(file)) { + assertThat("Couldn't find [" + file + "]", stream, notNullValue()); + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream) + ) { + // copyCurrentStructure does not property handle filters when it is passed a json parser. So we hide it. + return builder.copyCurrentStructure(new FilterXContentParser(parser) { + }); + } } }; } @@ -397,7 +398,7 @@ public void testBasics() throws Exception { } /** - * Generalization of {@link XContentMapValuesTests#testSupplementaryCharactersInPaths()} + * Tests that we can extract paths containing non-ascii characters. */ public void testFilterSupplementaryCharactersInPaths() throws IOException { Builder sample = builder -> builder.startObject().field("搜索", 2).field("指数", 3).endObject(); @@ -410,7 +411,7 @@ public void testFilterSupplementaryCharactersInPaths() throws IOException { } /** - * Generalization of {@link XContentMapValuesTests#testSharedPrefixes()} + * Tests that we can extract paths which share a prefix with other paths. */ public void testFilterSharedPrefixes() throws IOException { Builder sample = builder -> builder.startObject().field("foobar", 2).field("foobaz", 3).endObject(); @@ -423,7 +424,7 @@ public void testFilterSharedPrefixes() throws IOException { } /** - * Generalization of {@link XContentMapValuesTests#testPrefix()} + * Tests that we can extract paths which have another path as a prefix. */ public void testFilterPrefix() throws IOException { Builder sample = builder -> builder.startObject().array("photos", "foo", "bar").field("photosCount", 2).endObject(); @@ -447,10 +448,12 @@ public void testManyFilters() throws IOException, URISyntaxException { .endObject() .endObject() .endObject(); - Set manyFilters = Files.readAllLines( - PathUtils.get(AbstractFilteringTestCase.class.getResource("many_filters.txt").toURI()), - StandardCharsets.UTF_8 - ).stream().filter(s -> false == s.startsWith("#")).collect(toSet()); - testFilter(deep, deep, manyFilters, emptySet()); + try (InputStream stream = AbstractFilteringTestCase.class.getResourceAsStream("many_filters.txt")) { + assertThat("Couldn't find [many_filters.txt]", stream, notNullValue()); + Set manyFilters = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)).lines() + .filter(s -> false == s.startsWith("#")) + .collect(toSet()); + testFilter(deep, deep, manyFilters, emptySet()); + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 50079a920525f..bcc08edecb234 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.text.Text; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; @@ -174,7 +175,18 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { int numFields = randomInt(5); for (int i = 0; i < numFields; i++) { - builder.fetchField(randomAlphaOfLengthBetween(5, 10)); + String field = randomAlphaOfLengthBetween(5, 10); + String format = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null; + builder.fetchField(new FieldAndFormat(field, format)); + } + } + + if (randomBoolean()) { + int numFields = randomInt(5); + for (int i = 0; i < numFields; i++) { + String field = randomAlphaOfLengthBetween(5, 10); + String format = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null; + builder.docValueField(field, format); } } diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/many_filters.txt b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/many_filters.txt similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/many_filters.txt rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/many_filters.txt diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_authors.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_authors.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_authors.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_authors.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_authors_lastname.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_authors_lastname.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_authors_lastname.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_authors_lastname.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_pr.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_pr.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_pr.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_pr.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_distributors_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_distributors_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_distributors_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_distributors_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_no_distributors_name_no_street.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_no_distributors_name_no_street.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_no_distributors_name_no_street.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_en_no_distributors_name_no_street.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_no_distributors.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_no_distributors.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_no_distributors.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_properties_no_distributors.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_tags_authors_no_name.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_tags_authors_no_name.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_just_tags_authors_no_name.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_just_tags_authors_no_name.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_authors.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_authors.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_authors.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_authors.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_authors_lastname.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_authors_lastname.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_authors_lastname.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_authors_lastname.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_pr.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_pr.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_pr.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_pr.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_distributors_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_distributors_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_distributors_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_distributors_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_en_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_en_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_en_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_en_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_names.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_names.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_names.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_properties_names.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_tags.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_tags.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_tags.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_tags.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_title.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_title.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_title.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_title.json diff --git a/server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_title_pages.json b/test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_title_pages.json similarity index 100% rename from server/src/test/resources/org/elasticsearch/common/xcontent/support/sample_no_title_pages.json rename to test/framework/src/main/resources/org/elasticsearch/common/xcontent/support/sample_no_title_pages.json diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 0e6fbedd2c5e8..30bc47cbf5040 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -41,7 +41,6 @@ public class XPackLicenseState { * Each value defines the licensed state necessary for the feature to be allowed. */ public enum Feature { - SECURITY_AUDITING(OperationMode.GOLD, false), SECURITY_TOKEN_SERVICE(OperationMode.STANDARD, false), OPERATOR_PRIVILEGES(OperationMode.ENTERPRISE, true); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java index 32f0f36aa8c4b..ecf8c11fc4d47 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java @@ -86,16 +86,10 @@ public static OperationMode randomBasicStandardOrGold() { return randomFrom(BASIC, STANDARD, GOLD); } - public void testSecurityDefaults() { - XPackLicenseState licenseState = new XPackLicenseState(() -> 0); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true)); - } - public void testSecurityStandard() { XPackLicenseState licenseState = new XPackLicenseState(() -> 0); licenseState.update(STANDARD, true, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } @@ -103,7 +97,6 @@ public void testSecurityStandardExpired() { XPackLicenseState licenseState = new XPackLicenseState( () -> 0); licenseState.update(STANDARD, false, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } @@ -111,7 +104,6 @@ public void testSecurityBasic() { XPackLicenseState licenseState = new XPackLicenseState( () -> 0); licenseState.update(BASIC, true, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(false)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(false)); } @@ -119,7 +111,6 @@ public void testSecurityGold() { XPackLicenseState licenseState = new XPackLicenseState(() -> 0); licenseState.update(GOLD, true, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } @@ -127,7 +118,6 @@ public void testSecurityGoldExpired() { XPackLicenseState licenseState = new XPackLicenseState(() -> 0); licenseState.update(GOLD, false, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } @@ -135,7 +125,6 @@ public void testSecurityPlatinum() { XPackLicenseState licenseState = new XPackLicenseState(() -> 0); licenseState.update(PLATINUM, true, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } @@ -143,7 +132,6 @@ public void testSecurityPlatinumExpired() { XPackLicenseState licenseState = new XPackLicenseState(() -> 0); licenseState.update(PLATINUM, false, null); - assertThat(licenseState.checkFeature(Feature.SECURITY_AUDITING), is(true)); assertThat(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE), is(true)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java index c31dae68bf79b..299ce95ae4568 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java @@ -35,11 +35,12 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -108,7 +109,7 @@ public void setUpMocks() { clusterAdminClient = mock(ClusterAdminClient.class); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(new ClusterHealthResponse()); + listener.onResponse(new ClusterHealthResponse("", Strings.EMPTY_ARRAY, ClusterState.EMPTY_STATE, false)); return null; }).when(clusterAdminClient).health(any(ClusterHealthRequest.class), any(ActionListener.class)); diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlRestIT.java b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlClientYamlIT.java similarity index 82% rename from x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlRestIT.java rename to x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlClientYamlIT.java index 393c202520de2..521a1176fdbbe 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlRestIT.java +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/eql/EqlClientYamlIT.java @@ -12,9 +12,9 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -public class EqlRestIT extends ESClientYamlSuiteTestCase { +public class EqlClientYamlIT extends ESClientYamlSuiteTestCase { - public EqlRestIT(final ClientYamlTestCandidate testCandidate) { + public EqlClientYamlIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java index 4df04655d6e49..67cf4b1207f25 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java @@ -8,13 +8,12 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchContextMissingException; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -32,15 +31,16 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.junit.Before; import java.util.Collections; -import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField.AUTHORIZATION_INFO_KEY; import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField.ORIGINATING_ACTION_KEY; +import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; import static org.elasticsearch.xpack.security.authz.AuthorizationServiceTests.authzInfoRoles; import static org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener.ensureAuthenticatedUserIsSame; import static org.hamcrest.Matchers.is; @@ -98,8 +98,8 @@ public void testValidateSearchContext() throws Exception { new Authentication(new User("test", "role"), new RealmRef("realm", "file", "node"), null)); final IndicesAccessControl indicesAccessControl = mock(IndicesAccessControl.class); readerContext.putInContext(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); - XPackLicenseState licenseState = mock(XPackLicenseState.class); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + MockLicenseState licenseState = mock(MockLicenseState.class); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext); AuditTrail auditTrail = mock(AuditTrail.class); @@ -191,8 +191,8 @@ public void testEnsuredAuthenticatedUserIsSame() { ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong()); final String action = randomAlphaOfLength(4); TransportRequest request = Empty.INSTANCE; - XPackLicenseState licenseState = mock(XPackLicenseState.class); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + MockLicenseState licenseState = mock(MockLicenseState.class); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); AuditTrail auditTrail = mock(AuditTrail.class); AuditTrailService auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 0fef98af0f37e..9dc7c7065fb6a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -354,9 +354,9 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, // TODO: ip filtering does not actually track license usage yet public static final LicensedFeature.Momentary IP_FILTERING_FEATURE = - LicensedFeature.momentaryLenient(null, "security_ip_filtering", License.OperationMode.GOLD); + LicensedFeature.momentaryLenient(null, "security-ip-filtering", License.OperationMode.GOLD); public static final LicensedFeature.Momentary AUDITING_FEATURE = - LicensedFeature.momentaryLenient(null, "security_auditing", License.OperationMode.GOLD); + LicensedFeature.momentaryLenient(null, "security-auditing", License.OperationMode.GOLD); private static final String REALMS_FEATURE_FAMILY = "security-realms"; // Builtin realms (file/native) realms are Basic licensed, so don't need to be checked or tracked diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java index 43945e30e0987..e888a2db910f1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java @@ -10,13 +10,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import java.net.InetAddress; @@ -43,7 +43,7 @@ public AuditTrailService(List auditTrails, XPackLicenseState license public AuditTrail get() { if (compositeAuditTrail.isEmpty() == false) { - if (licenseState.checkFeature(Feature.SECURITY_AUDITING)) { + if (Security.AUDITING_FEATURE.check(licenseState)) { return compositeAuditTrail; } else { maybeLogAuditingDisabled(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java index a585206585679..f0e97f7e87569 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java @@ -11,8 +11,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.license.License; -import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; @@ -22,6 +21,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.Before; @@ -47,7 +47,7 @@ public class AuditTrailServiceTests extends ESTestCase { private AuthenticationToken token; private TransportRequest request; private RestRequest restRequest; - private XPackLicenseState licenseState; + private MockLicenseState licenseState; private boolean isAuditingAllowed; @Before @@ -57,10 +57,10 @@ public void init() throws Exception { auditTrailsBuilder.add(mock(AuditTrail.class)); } auditTrails = unmodifiableList(auditTrailsBuilder); - licenseState = mock(XPackLicenseState.class); + licenseState = mock(MockLicenseState.class); service = new AuditTrailService(auditTrails, licenseState); isAuditingAllowed = randomBoolean(); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(isAuditingAllowed); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(isAuditingAllowed); token = mock(AuthenticationToken.class); request = mock(TransportRequest.class); restRequest = mock(RestRequest.class); @@ -118,7 +118,7 @@ public void testNoLogRecentlyWhenLicenseProhibitsAuditing() throws Exception { public void testAuthenticationFailed() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, token, "_action", request); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, token, "_action", request); @@ -131,7 +131,7 @@ public void testAuthenticationFailed() throws Exception { public void testAuthenticationFailedNoToken() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, "_action", request); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, "_action", request); @@ -144,7 +144,7 @@ public void testAuthenticationFailedNoToken() throws Exception { public void testAuthenticationFailedRestNoToken() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, restRequest); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, restRequest); @@ -157,7 +157,7 @@ public void testAuthenticationFailedRestNoToken() throws Exception { public void testAuthenticationFailedRest() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, token, restRequest); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, token, restRequest); @@ -170,7 +170,7 @@ public void testAuthenticationFailedRest() throws Exception { public void testAuthenticationFailedRealm() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, "_realm", token, "_action", request); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, "_realm", token, "_action", request); @@ -183,7 +183,7 @@ public void testAuthenticationFailedRealm() throws Exception { public void testAuthenticationFailedRestRealm() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationFailed(requestId, "_realm", token, restRequest); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationFailed(requestId, "_realm", token, restRequest); @@ -196,7 +196,7 @@ public void testAuthenticationFailedRestRealm() throws Exception { public void testAnonymousAccess() throws Exception { final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().anonymousAccessDenied(requestId, "_action", request); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).anonymousAccessDenied(requestId, "_action", request); @@ -213,7 +213,7 @@ public void testAccessGranted() throws Exception { () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) }); final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().accessGranted(requestId, authentication, "_action", request, authzInfo); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).accessGranted(requestId, authentication, "_action", request, authzInfo); @@ -230,7 +230,7 @@ public void testAccessDenied() throws Exception { () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) }); final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().accessDenied(requestId, authentication, "_action", request, authzInfo); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).accessDenied(requestId, authentication, "_action", request, authzInfo); @@ -244,7 +244,7 @@ public void testConnectionGranted() throws Exception { InetAddress inetAddress = InetAddress.getLoopbackAddress(); SecurityIpFilterRule rule = randomBoolean() ? SecurityIpFilterRule.ACCEPT_ALL : IPFilter.DEFAULT_PROFILE_ACCEPT_ALL; service.get().connectionGranted(inetAddress, "client", rule); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).connectionGranted(inetAddress, "client", rule); @@ -258,7 +258,7 @@ public void testConnectionDenied() throws Exception { InetAddress inetAddress = InetAddress.getLoopbackAddress(); SecurityIpFilterRule rule = new SecurityIpFilterRule(false, "_all"); service.get().connectionDenied(inetAddress, "client", rule); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).connectionDenied(inetAddress, "client", rule); @@ -273,7 +273,7 @@ public void testAuthenticationSuccessRest() throws Exception { new RealmRef(null, null, null)); final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationSuccess(requestId, authentication, restRequest); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationSuccess(requestId, authentication, restRequest); @@ -288,7 +288,7 @@ public void testAuthenticationSuccessTransport() throws Exception { new RealmRef(null, null, null)); final String requestId = randomAlphaOfLengthBetween(6, 12); service.get().authenticationSuccess(requestId, authentication, "_action", request); - verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); + verify(licenseState).isAllowed(Security.AUDITING_FEATURE); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { verify(auditTrail).authenticationSuccess(requestId, authentication, "_action", request); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 518e9815a11c4..9dc8c97bb898b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -233,7 +233,7 @@ public void init() throws Exception { when(licenseState.isAllowed(Security.CUSTOM_REALMS_FEATURE)).thenReturn(true); when(licenseState.checkFeature(Feature.SECURITY_TOKEN_SERVICE)).thenReturn(true); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); when(licenseState.getOperationMode()).thenReturn(randomFrom(License.OperationMode.ENTERPRISE, License.OperationMode.PLATINUM)); ReservedRealm reservedRealm = mock(ReservedRealm.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 35388664ec3cf..4522f3487807b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -60,8 +60,12 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollAction; import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClosePointInTimeAction; +import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.MultiSearchAction; import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.OpenPointInTimeAction; +import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.ParsedScrollId; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; @@ -88,23 +92,21 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; -import org.elasticsearch.license.MockLicenseState; -import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.bulk.stats.BulkOperationListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.TestIndexNameExpressionResolver; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ShardSearchRequest; @@ -114,10 +116,7 @@ import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.action.search.ClosePointInTimeAction; -import org.elasticsearch.action.search.ClosePointInTimeRequest; -import org.elasticsearch.action.search.OpenPointInTimeAction; -import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; @@ -203,8 +202,8 @@ import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField.ORIGINATING_ACTION_KEY; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; -import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.RESTRICTED_INDICES_AUTOMATON; +import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; @@ -253,7 +252,7 @@ public void setup() { when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); auditTrail = mock(AuditTrail.class); MockLicenseState licenseState = mock(MockLicenseState.class); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState); threadContext = new ThreadContext(settings); threadPool = mock(ThreadPool.class); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java index 85f49df70447b..153d6f38528da 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.MockLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; @@ -30,6 +29,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditTrailService; import java.util.Collections; @@ -51,7 +51,7 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase { public void testInterceptorThrowsWhenFLSDLSEnabled() { MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState); @@ -110,7 +110,7 @@ public void testInterceptorThrowsWhenFLSDLSEnabled() { public void testInterceptorThrowsWhenTargetHasGreaterPermissions() throws Exception { MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(randomBoolean()); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java index 6b57e38c641be..a7ced96da4853 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.MockLicenseState; -import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.authc.Authentication; @@ -35,6 +34,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.support.Automatons; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditTrailService; import java.util.Collections; @@ -56,7 +56,7 @@ public class ResizeRequestInterceptorTests extends ESTestCase { public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() { MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true); ThreadPool threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); @@ -108,7 +108,7 @@ public void testResizeRequestInterceptorThrowsWhenFLSDLSEnabled() { public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions() throws Exception { MockLicenseState licenseState = mock(MockLicenseState.class); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.checkFeature(Feature.SECURITY_AUDITING)).thenReturn(true); + when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true); when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true); ThreadPool threadPool = mock(ThreadPool.class); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/40_knn_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/40_knn_search.yml new file mode 100644 index 0000000000000..a83ec8c410978 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/40_knn_search.yml @@ -0,0 +1,90 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_replicas: 0 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + - do: + index: + index: test + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555, -200.0] + + - do: + index: + index: test + id: 2 + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8, -156.0] + + - do: + index: + index: test + id: 3 + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - do: + indices.refresh: {} + +--- +"Basic kNN search": + - do: + knn_search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + +--- +"Test nonexistent field": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + knn: + field: nonexistent + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + num_candidates: 1 + - match: { error.root_cause.0.type: "illegal_argument_exception" } + +--- +"Direct knn queries are disallowed": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + num_candidates: 1 + - match: { error.root_cause.0.type: "illegal_argument_exception" } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java index 050c5f6294239..b9c759783745c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java @@ -188,7 +188,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenShardInitializationPen doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(new ClusterHealthResponse()); + listener.onResponse(new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, false)); return null; }).when(clusterClient).health(any(), any()); @@ -272,7 +272,7 @@ public void testCreateLatestVersionedIndexIfRequired_GivenConcurrentCreationShar doAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; - listener.onResponse(new ClusterHealthResponse()); + listener.onResponse(new ClusterHealthResponse("", new String[]{}, ClusterState.EMPTY_STATE, false)); return null; }).when(clusterClient).health(any(), any()); diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/DenseVectorPlugin.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/DenseVectorPlugin.java index cdd921c79603f..201c06a76fda4 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/DenseVectorPlugin.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/DenseVectorPlugin.java @@ -7,17 +7,31 @@ package org.elasticsearch.xpack.vectors; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.xpack.vectors.action.RestKnnSearchAction; import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper; import org.elasticsearch.xpack.vectors.mapper.SparseVectorFieldMapper; +import org.elasticsearch.xpack.vectors.query.KnnVectorQueryBuilder; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.function.Supplier; -public class DenseVectorPlugin extends Plugin implements MapperPlugin { +public class DenseVectorPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin { public DenseVectorPlugin() { } @@ -28,4 +42,26 @@ public Map getMappers() { mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, SparseVectorFieldMapper.PARSER); return Collections.unmodifiableMap(mappers); } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(new RestKnnSearchAction()); + } + + @Override + public List> getQueries() { + // This query is only meant to be used internally, and not passed to the _search endpoint + return List.of(new QuerySpec<>(KnnVectorQueryBuilder.NAME, KnnVectorQueryBuilder::new, + parser -> { + throw new IllegalArgumentException("[knn] queries cannot be provided directly, use the [_knn_search] endpoint instead"); + })); + } } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilder.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilder.java new file mode 100644 index 0000000000000..b69d2f3f2ed3b --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilder.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectors.action; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.vectors.query.KnnVectorQueryBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +/** + * A builder used in {@link RestKnnSearchAction} to convert the kNN REST request + * into a {@link SearchRequestBuilder}. + */ +class KnnSearchRequestBuilder { + static final String INDEX_PARAM = "index"; + static final String ROUTING_PARAM = "routing"; + + static final ParseField KNN_SECTION_FIELD = new ParseField("knn"); + private static final ObjectParser PARSER; + + static { + PARSER = new ObjectParser<>("knn-search"); + PARSER.declareField(KnnSearchRequestBuilder::knnSearch, KnnSearch::parse, + KNN_SECTION_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField((p, request, c) -> request.fetchSource(FetchSourceContext.fromXContent(p)), + SearchSourceBuilder._SOURCE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING); + PARSER.declareFieldArray(KnnSearchRequestBuilder::fields, (p, c) -> FieldAndFormat.fromXContent(p), + SearchSourceBuilder.FETCH_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); + PARSER.declareFieldArray(KnnSearchRequestBuilder::docValueFields, (p, c) -> FieldAndFormat.fromXContent(p), + SearchSourceBuilder.DOCVALUE_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); + PARSER.declareField((p, request, c) -> request.storedFields( + StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), p)), + SearchSourceBuilder.STORED_FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY); + } + + /** + * Parses a {@link RestRequest} representing a kNN search into a request builder. + */ + static KnnSearchRequestBuilder parseRestRequest(RestRequest restRequest) throws IOException { + KnnSearchRequestBuilder builder = new KnnSearchRequestBuilder( + Strings.splitStringByCommaToArray(restRequest.param("index"))); + builder.routing(restRequest.param("routing")); + + if (restRequest.hasContentOrSourceParam()) { + try (XContentParser contentParser = restRequest.contentOrSourceParamParser()) { + PARSER.parse(contentParser, builder, null); + } + } + return builder; + } + + private final String[] indices; + private String routing; + private KnnSearch knnSearch; + + private FetchSourceContext fetchSource; + private List fields; + private List docValueFields; + private StoredFieldsContext storedFields; + + private KnnSearchRequestBuilder(String[] indices) { + this.indices = indices; + } + + /** + * Defines the kNN search to execute. + */ + private void knnSearch(KnnSearch knnSearch) { + this.knnSearch = knnSearch; + } + + /** + * A comma separated list of routing values to control the shards the search will be executed on. + */ + private void routing(String routing) { + this.routing = routing; + } + + /** + * Defines how the _source should be fetched. + */ + private void fetchSource(FetchSourceContext fetchSource) { + this.fetchSource = fetchSource; + } + + /** + * A list of fields to load and return. The fields must be present in the document _source. + */ + private void fields(List fields) { + this.fields = fields; + } + + /** + * A list of docvalue fields to load and return. + */ + private void docValueFields(List docValueFields) { + this.docValueFields = docValueFields; + } + + /** + * Defines the stored fields to load and return as part of the search request. To disable the stored + * fields entirely (source and metadata fields), use {@link StoredFieldsContext#_NONE_}. + */ + private void storedFields(StoredFieldsContext storedFields) { + this.storedFields = storedFields; + } + + /** + * Adds all the request components to the given {@link SearchRequestBuilder}. + */ + public void build(SearchRequestBuilder builder) { + builder.setIndices(indices); + builder.setRouting(routing); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_ACCURATE); + + if (knnSearch == null) { + throw new IllegalArgumentException("missing required [" + KNN_SECTION_FIELD.getPreferredName() + "] section in search body"); + } + knnSearch.build(sourceBuilder); + + sourceBuilder.fetchSource(fetchSource); + sourceBuilder.storedFields(storedFields); + + if (fields != null) { + for (FieldAndFormat field : fields) { + sourceBuilder.fetchField(field); + } + } + + if (docValueFields != null) { + for (FieldAndFormat field : docValueFields) { + sourceBuilder.docValueField(field.field, field.format); + } + } + + builder.setSource(sourceBuilder); + } + + // visible for testing + static class KnnSearch { + private static final int NUM_CANDS_LIMIT = 10000; + static final ParseField FIELD_FIELD = new ParseField("field"); + static final ParseField K_FIELD = new ParseField("k"); + static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates"); + static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("knn", args -> { + @SuppressWarnings("unchecked") + List vector = (List) args[1]; + float[] vectorArray = new float[vector.size()]; + for (int i = 0; i < vector.size(); i++) { + vectorArray[i] = vector.get(i); + } + return new KnnSearch((String) args[0], vectorArray, (int) args[2], (int) args[3]); + }); + + static { + PARSER.declareString(constructorArg(), FIELD_FIELD); + PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD); + PARSER.declareInt(constructorArg(), K_FIELD); + PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD); + } + + public static KnnSearch parse(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + final String field; + final float[] queryVector; + final int k; + final int numCands; + + /** + * Defines a kNN search. + * + * @param field the name of the vector field to search against + * @param queryVector the query vector + * @param k the final number of nearest neighbors to return as top hits + * @param numCands the number of nearest neighbor candidates to consider per shard + */ + KnnSearch(String field, float[] queryVector, int k, int numCands) { + this.field = field; + this.queryVector = queryVector; + this.k = k; + this.numCands = numCands; + } + + void build(SearchSourceBuilder builder) { + // We perform validation here instead of the constructor because it makes the errors + // much clearer. Otherwise, the error message is deeply nested under parsing exceptions. + if (k < 1) { + throw new IllegalArgumentException("[" + K_FIELD.getPreferredName() + "] must be greater than 0"); + } + if (numCands < k) { + throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot be less than " + + "[" + K_FIELD.getPreferredName() + "]"); + } + if (numCands > NUM_CANDS_LIMIT) { + throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]"); + } + + builder.query(new KnnVectorQueryBuilder(field, queryVector, numCands)); + builder.size(k); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KnnSearch that = (KnnSearch) o; + return k == that.k && numCands == that.numCands + && Objects.equals(field, that.field) && Arrays.equals(queryVector, that.queryVector); + } + + @Override + public int hashCode() { + int result = Objects.hash(field, k, numCands); + result = 31 * result + Arrays.hashCode(queryVector); + return result; + } + } +} diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/RestKnnSearchAction.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/RestKnnSearchAction.java new file mode 100644 index 0000000000000..befe73d225865 --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/action/RestKnnSearchAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.vectors.action; + + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestStatusToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; + +/** + * The REST action for handling kNN searches. Currently, it just parses + * the REST request into a search request and calls the search action. + */ +public class RestKnnSearchAction extends BaseRestHandler { + + public RestKnnSearchAction() {} + + @Override + public List routes() { + return List.of( + new Route(GET, "{index}/_knn_search"), + new Route(POST, "{index}/_knn_search")); + } + + @Override + public String getName() { + return "knn_search_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + // This will allow to cancel the search request if the http channel is closed + RestCancellableNodeClient cancellableNodeClient = new RestCancellableNodeClient(client, restRequest.getHttpChannel()); + KnnSearchRequestBuilder request = KnnSearchRequestBuilder.parseRestRequest(restRequest); + + SearchRequestBuilder searchRequestBuilder = cancellableNodeClient.prepareSearch(); + request.build(searchRequestBuilder); + + return channel -> searchRequestBuilder.execute(new RestStatusToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 51f25db3fecc6..61b899f2c5fb6 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -204,6 +204,10 @@ public DenseVectorFieldType(String name, Version indexVersionCreated, int dims, this.indexVersionCreated = indexVersionCreated; } + public int dims() { + return dims; + } + @Override public String typeName() { return CONTENT_TYPE; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilder.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilder.java new file mode 100644 index 0000000000000..226a017f0d8dc --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilder.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.xpack.vectors.query; + +import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper; +import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper.DenseVectorFieldType; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +public class KnnVectorQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "knn"; + + private final String fieldName; + private final float[] queryVector; + private final int numCands; + + public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands) { + this.fieldName = fieldName; + this.queryVector = queryVector; + this.numCands = numCands; + } + + public KnnVectorQueryBuilder(StreamInput in) throws IOException { + super(in); + this.fieldName = in.readString(); + this.numCands = in.readVInt(); + this.queryVector = in.readFloatArray(); + } + + public String getFieldName() { + return fieldName; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(fieldName); + out.writeVInt(numCands); + out.writeFloatArray(queryVector); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME) + .field("field", fieldName) + .field("vector", queryVector) + .field("num_candidates", numCands); + builder.endObject(); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected Query doToQuery(SearchExecutionContext context) { + MappedFieldType fieldType = context.getFieldType(fieldName); + if (fieldType == null) { + throw new IllegalArgumentException("field [" + fieldName + "] does not exist in the mapping"); + } + + if (fieldType instanceof DenseVectorFieldType == false) { + throw new IllegalArgumentException("[" + NAME + "] queries are only supported on [" + + DenseVectorFieldMapper.CONTENT_TYPE + "] fields"); + } + + DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType; + if (queryVector.length != vectorFieldType.dims()) { + throw new IllegalArgumentException("the query vector has a different dimension [" + queryVector.length + "] " + + "than the index vectors [" + vectorFieldType.dims() + "]"); + } + if (vectorFieldType.isSearchable() == false) { + throw new IllegalArgumentException("[" + "[" + NAME + "] queries are not supported if [index] is disabled"); + } + return new KnnVectorQuery(fieldType.name(), queryVector, numCands); + } + + @Override + protected int doHashCode() { + return Objects.hash(fieldName, Arrays.hashCode(queryVector), numCands); + } + + @Override + protected boolean doEquals(KnnVectorQueryBuilder other) { + return Objects.equals(fieldName, other.fieldName) && + Arrays.equals(queryVector, other.queryVector) && + numCands == other.numCands; + } +} diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilderTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilderTests.java new file mode 100644 index 0000000000000..183393a9b2970 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/action/KnnSearchRequestBuilderTests.java @@ -0,0 +1,254 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectors.action; + +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.vectors.action.KnnSearchRequestBuilder.KnnSearch; +import org.elasticsearch.xpack.vectors.query.KnnVectorQueryBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder; +import static org.hamcrest.Matchers.containsString; + +public class KnnSearchRequestBuilderTests extends ESTestCase { + + public void testBuildSearchRequest() throws IOException { + // Choose random REST parameters + Map params = new HashMap<>(); + String[] indices = generateRandomStringArray(5, 10, false, true); + params.put(KnnSearchRequestBuilder.INDEX_PARAM, String.join(",", indices)); + + String routing = null; + if (randomBoolean()) { + routing = randomAlphaOfLengthBetween(3, 10); + params.put(KnnSearchRequestBuilder.ROUTING_PARAM, routing); + } + + // Create random request body + KnnSearch knnSearch = randomKnnSearch(); + SearchSourceBuilder searchSource = randomSearchSourceBuilder( + () -> null, + () -> null, + () -> null, + Collections::emptyList, + () -> null, + () -> null); + XContentBuilder builder = createRequestBody(knnSearch, searchSource); + + // Convert the REST request to a search request and check the components + SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder, params); + SearchRequest searchRequest = searchRequestBuilder.request(); + + assertArrayEquals(indices, searchRequest.indices()); + assertEquals(routing, searchRequest.routing()); + + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(knnSearch.field, knnSearch.queryVector, knnSearch.numCands); + assertEquals(query, searchRequest.source().query()); + assertEquals(knnSearch.k, searchRequest.source().size()); + + assertEquals(searchSource.fetchSource(), searchRequest.source().fetchSource()); + assertEquals(searchSource.fetchFields(), searchRequest.source().fetchFields()); + assertEquals(searchSource.docValueFields(), searchRequest.source().docValueFields()); + assertEquals(searchSource.storedFields(), searchRequest.source().storedFields()); + } + + public void testParseSourceString() throws IOException { + // Create random request body + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + + KnnSearch knnSearch = randomKnnSearch(); + builder.startObject() + .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) + .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) + .endObject(); + + builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), "some-field"); + builder.endObject(); + + // Convert the REST request to a search request and check the components + SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder); + SearchRequest searchRequest = searchRequestBuilder.request(); + + FetchSourceContext fetchSource = searchRequest.source().fetchSource(); + assertTrue(fetchSource.fetchSource()); + assertArrayEquals(new String[]{"some-field"}, fetchSource.includes()); + } + + public void testParseSourceArray() throws IOException { + // Create random request body + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + + KnnSearch knnSearch = randomKnnSearch(); + builder.startObject() + .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) + .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) + .endObject(); + + builder.array(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), "field1", "field2", "field3"); + builder.endObject(); + + // Convert the REST request to a search request and check the components + SearchRequestBuilder searchRequestBuilder = buildSearchRequest(builder); + SearchRequest searchRequest = searchRequestBuilder.request(); + + FetchSourceContext fetchSource = searchRequest.source().fetchSource(); + assertTrue(fetchSource.fetchSource()); + assertArrayEquals(new String[]{"field1", "field2", "field3"}, fetchSource.includes()); + } + + public void testMissingKnnSection() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()).startObject() + .array(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName(), "field1", "field2") + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + assertThat(e.getMessage(), containsString("missing required [knn] section in search body")); + } + + public void testNumCandsLessThanK() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()).startObject() + .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") + .field(KnnSearch.K_FIELD.getPreferredName(), 100) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 80) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[]{1.0f, 2.0f, 3.0f}) + .endObject() + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + assertThat(e.getMessage(), containsString("[num_candidates] cannot be less than [k]")); + } + + public void testNumCandsExceedsLimit() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()).startObject() + .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") + .field(KnnSearch.K_FIELD.getPreferredName(), 100) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10002) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[]{1.0f, 2.0f, 3.0f}) + .endObject() + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + assertThat(e.getMessage(), containsString("[num_candidates] cannot exceed [10000]")); + } + + public void testInvalidK() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()).startObject() + .startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), "field") + .field(KnnSearch.K_FIELD.getPreferredName(), 0) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), 10) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), new float[]{1.0f, 2.0f, 3.0f}) + .endObject() + .endObject(); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildSearchRequest(builder)); + assertThat(e.getMessage(), containsString("[k] must be greater than 0")); + } + + private SearchRequestBuilder buildSearchRequest(XContentBuilder builder) throws IOException { + Map params = Map.of(KnnSearchRequestBuilder.INDEX_PARAM, "index"); + return buildSearchRequest(builder, params); + } + + private SearchRequestBuilder buildSearchRequest(XContentBuilder builder, Map params) throws IOException { + KnnSearchRequestBuilder knnRequestBuilder = KnnSearchRequestBuilder.parseRestRequest( + new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.POST) + .withParams(params) + .withContent(BytesReference.bytes(builder), builder.contentType()) + .build()); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(null, SearchAction.INSTANCE); + knnRequestBuilder.build(searchRequestBuilder); + return searchRequestBuilder; + } + + private KnnSearch randomKnnSearch() { + String field = randomAlphaOfLength(6); + int dim = randomIntBetween(2, 30); + float[] vector = new float[dim]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + + int k = randomIntBetween(1, 100); + int numCands = randomIntBetween(k, 1000); + return new KnnSearch(field, vector, k, numCands); + } + + private XContentBuilder createRequestBody(KnnSearch knnSearch, SearchSourceBuilder searchSource) throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + builder.startObject(); + + builder.startObject(KnnSearchRequestBuilder.KNN_SECTION_FIELD.getPreferredName()) + .field(KnnSearch.FIELD_FIELD.getPreferredName(), knnSearch.field) + .field(KnnSearch.K_FIELD.getPreferredName(), knnSearch.k) + .field(KnnSearch.NUM_CANDS_FIELD.getPreferredName(), knnSearch.numCands) + .field(KnnSearch.QUERY_VECTOR_FIELD.getPreferredName(), knnSearch.queryVector) + .endObject(); + + if (searchSource.fetchSource() != null) { + builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName()); + searchSource.fetchSource().toXContent(builder, ToXContent.EMPTY_PARAMS); + } + + if (searchSource.fetchFields() != null) { + builder.startArray(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName()); + for (FieldAndFormat fieldAndFormat : searchSource.fetchFields()) { + fieldAndFormat.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endArray(); + } + + if (searchSource.docValueFields() != null) { + builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName()); + for (FieldAndFormat fieldAndFormat : searchSource.docValueFields()) { + fieldAndFormat.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endArray(); + } + + if (searchSource.storedFields() != null) { + searchSource.storedFields().toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder); + } + + builder.endObject(); + return builder; + } + +} diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnSearchActionTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnSearchActionTests.java new file mode 100644 index 0000000000000..0a8235204ce93 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnSearchActionTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectors.query; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.vectors.DenseVectorPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; + +public class KnnSearchActionTests extends ESSingleNodeTestCase { + private static final int VECTOR_DIMENSION = 10; + + @Override + protected Collection> getPlugins() { + return List.of(DenseVectorPlugin.class); + } + + public void testTotalHits() throws IOException { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject("vector") + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .endObject().endObject(); + createIndex("index1", indexSettings, builder); + createIndex("index2", indexSettings, builder); + + for (int doc = 0; doc < 10; doc++) { + client().prepareIndex("index1") + .setId(String.valueOf(doc)) + .setSource("vector", randomVector()) + .get(); + client().prepareIndex("index2") + .setId(String.valueOf(doc)) + .setSource("vector", randomVector()) + .get(); + } + + client().admin().indices().prepareForceMerge("index1", "index2").setMaxNumSegments(1).get(); + client().admin().indices().prepareRefresh("index1", "index2").get(); + + // Since there's no kNN search action at the transport layer, we just emulate + // how the action works (it builds a kNN query under the hood) + float[] queryVector = randomVector(); + SearchResponse response = client().prepareSearch("index1", "index2") + .setQuery(new KnnVectorQueryBuilder("vector", queryVector, 5)) + .setSize(2) + .get(); + + // The total hits is num_cands * num_shards, since the query gathers num_cands hits from each shard + assertHitCount(response, 5 * 2); + assertEquals(2, response.getHits().getHits().length); + } + + private float[] randomVector() { + float[] vector = new float[VECTOR_DIMENSION]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + return vector; + } +} diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilderTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilderTests.java new file mode 100644 index 0000000000000..548ae62410977 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/KnnVectorQueryBuilderTests.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectors.query; + +import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.AbstractBuilderTestCase; +import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.vectors.DenseVectorPlugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import static org.hamcrest.Matchers.containsString; + +public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase { + private static final String VECTOR_FIELD = "vector"; + private static final String VECTOR_ALIAS_FIELD = "vector_alias"; + private static final String UNINDEXED_VECTOR_FIELD = "unindexed_vector"; + private static final int VECTOR_DIMENSION = 3; + + @Override + protected Collection> getPlugins() { + return Arrays.asList(DenseVectorPlugin.class, TestGeoShapeFieldMapperPlugin.class); + } + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties") + .startObject(VECTOR_FIELD) + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .field("index", true) + .field("similarity", "l2_norm") + .endObject() + .startObject(VECTOR_ALIAS_FIELD) + .field("type", "alias") + .field("path", VECTOR_FIELD) + .endObject() + .startObject(UNINDEXED_VECTOR_FIELD) + .field("type", "dense_vector") + .field("dims", VECTOR_DIMENSION) + .endObject() + .endObject().endObject(); + mapperService.merge(MapperService.SINGLE_MAPPING_NAME, + new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @Override + protected KnnVectorQueryBuilder doCreateTestQueryBuilder() { + String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD; + + float[] vector = new float[VECTOR_DIMENSION]; + for (int i = 0; i < vector.length; i++) { + vector[i] = randomFloat(); + } + int numCands = randomIntBetween(1, 1000); + return new KnnVectorQueryBuilder(fieldName, vector, numCands); + } + + @Override + protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) { + // TODO: expose getters on KnnVectorQuery and assert more here + assertTrue(query instanceof KnnVectorQuery); + } + + public void testWrongDimension() { + SearchExecutionContext context = createSearchExecutionContext(); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] {1.0f, 2.0f}, 10); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); + assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]")); + } + + public void testNonexistentField() { + SearchExecutionContext context = createSearchExecutionContext(); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", + new float[]{1.0f, 1.0f, 1.0f}, 10); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); + assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping")); + } + + public void testWrongFieldType() { + SearchExecutionContext context = createSearchExecutionContext(); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(AbstractBuilderTestCase.KEYWORD_FIELD_NAME, + new float[]{1.0f, 1.0f, 1.0f}, 10); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); + assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields")); + } + + public void testUnindexedField() { + SearchExecutionContext context = createSearchExecutionContext(); + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(UNINDEXED_VECTOR_FIELD, + new float[]{1.0f, 1.0f, 1.0f}, 10); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context)); + assertThat(e.getMessage(), containsString("[knn] queries are not supported if [index] is disabled")); + } + + @Override + public void testValidOutput() { + KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] {1.0f, 2.0f, 3.0f}, 10); + String expected = "{\n" + + " \"knn\" : {\n" + + " \"field\" : \"vector\",\n" + + " \"vector\" : [\n" + + " 1.0,\n" + + " 2.0,\n" + + " 3.0\n" + + " ],\n" + + " \"num_candidates\" : 10\n" + + " }\n" + + "}"; + assertEquals(expected, query.toString()); + } + + @Override + public void testUnknownObjectException() throws IOException { + // Test isn't relevant, since query is never parsed from xContent + } + + @Override + public void testFromXContent() throws IOException { + // Test isn't relevant, since query is never parsed from xContent + } + + @Override + public void testUnknownField() throws IOException { + // Test isn't relevant, since query is never parsed from xContent + } +}