From ddf615272b91d4bf99a38f0217195c52728e298c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 26 Mar 2021 16:24:16 +0100 Subject: [PATCH 01/32] transformation working --- .../compat/RestCompatTestTransformTask.java | 5 + .../rest/transform/RestTestTransformer.java | 9 +- .../rest/transform/match/ReplaceTextual.java | 76 +++++++++++ .../transform/match/ReplaceTextualTests.java | 121 ++++++++++++++++++ .../rest/transform/match/is_true.yml | 110 ++++++++++++++++ rest-api-spec/build.gradle | 12 +- .../admin/indices/get/GetIndexResponse.java | 22 +++- .../mapping/get/GetMappingsResponse.java | 4 + .../elasticsearch/rest/BaseRestHandler.java | 21 ++- .../admin/indices/RestCreateIndexAction.java | 65 +++++++++- .../admin/indices/RestGetIndicesAction.java | 27 ++++ .../indices/RestCreateIndexActionTests.java | 69 ++++++++++ .../org/elasticsearch/test/ESTestCase.java | 4 + 13 files changed, 532 insertions(+), 13 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java create mode 100644 buildSrc/src/test/resources/rest/transform/match/is_true.yml diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 26cf4664d3ac7..ab7bf3ae218c2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -23,6 +23,7 @@ import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; +import org.elasticsearch.gradle.test.rest.transform.match.ReplaceTextual; import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; import org.elasticsearch.gradle.test.rest.transform.warnings.InjectAllowedWarnings; import org.elasticsearch.gradle.test.rest.transform.warnings.InjectWarnings; @@ -101,6 +102,10 @@ public void replaceMatch(String subKey, Object value) { transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } + public void replaceIsTrue(String subKey, Object value) { + transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, JsonNode.class))); + } + /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java index 5f276d75257dc..2003397acd8af 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java @@ -144,9 +144,16 @@ private void traverseTest( } else { if (entry.getValue().isObject()) { ObjectNode child = (ObjectNode) entry.getValue(); - if (child.has(transform.requiredChildKey())) { +// if (child.has(transform.requiredChildKey())) { + if(transform.matches(child)) { transform.transformTest((ObjectNode) currentNode); } + } else if(entry.getValue().isTextual()){ + String fieldValue = entry.getValue().asText(); + if(transform.matches(fieldValue)) {///fieldValue.equals(transform.requiredChildKey())){ + transform.transformTest((ObjectNode) currentNode); + } + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java new file mode 100644 index 0000000000000..86a06d4df9158 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +/** + * A transformation to replace the flat fields + */ +public class ReplaceTextual implements RestTestTransformByParentObject { + private final String replaceKey; + private final JsonNode replacementNode; + private final String testName; + + public ReplaceTextual(String replaceKey, JsonNode replacementNode) { + this.replaceKey = replaceKey; + this.replacementNode = replacementNode; + this.testName = null; + } + + public ReplaceTextual(String replaceKey, JsonNode replacementNode, String testName) { + this.replaceKey = replaceKey; + this.replacementNode = replacementNode; + this.testName = testName; + } + + @Override + @Internal + public String getKeyToFind() { + return "is_true"; + } + + @Override + public String requiredChildKey() { + return replaceKey; + } + + @Override + public boolean shouldApply(RestTestContext testContext) { + return testName == null || testContext.getTestName().equals(testName); + } + + @Override + public void transformTest(ObjectNode matchParent) { +// ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + matchParent.set(getKeyToFind(), replacementNode); + } + + @Input + public String getReplaceKey() { + return replaceKey; + } + + @Input + public JsonNode getReplacementNode() { + return replacementNode; + } + + @Input + @Optional + public String getTestName() { + return testName; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java new file mode 100644 index 0000000000000..07aa62adc82dc --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.elasticsearch.gradle.test.rest.transform.TransformTests; +import org.hamcrest.CoreMatchers; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ReplaceTextualTests extends TransformTests { + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + + @Test + public void testReplaceAll() throws Exception { + String testName = "/rest/transform/match/is_true.yml"; + List tests = getTests(testName); + JsonNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", JsonNode.class); + validateTest(tests, true, true); + List transformedTests = transformTests( + new LinkedList<>(tests), + Collections.singletonList(new ReplaceTextual("test_index.mappings.type_1", replacementNode, null)) + ); + printTest(testName, transformedTests); + validateTest(tests, false, true); + + } + + private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { + validateSetupAndTearDownForMatchTests(tests); + // first test + JsonNode firstTestChild = tests.get(2).get("First test"); + assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; + + AtomicBoolean firstTestHasMatchObject = new AtomicBoolean(false); + AtomicBoolean firstTestHasTypeMatch = new AtomicBoolean(false); + + firstTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("is_true"); + if (matchObject != null) { + firstTestHasMatchObject.set(true); + if (firstTestHasTypeMatch.get() == false ) { + firstTestHasTypeMatch.set(true); + }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + } + } + }); + assertTrue(firstTestHasMatchObject.get()); + assertTrue(firstTestHasTypeMatch.get()); + + // last test + JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); + assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; + + AtomicBoolean lastTestHasMatchObject = new AtomicBoolean(false); + AtomicBoolean lastTestHasTypeMatch = new AtomicBoolean(false); + lastTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("is_true"); + if (matchObject != null) { + lastTestHasMatchObject.set(true); + if (lastTestHasTypeMatch.get() == false ) { + lastTestHasTypeMatch.set(true); + }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + } + } + }); + assertTrue(lastTestHasMatchObject.get()); + assertTrue(lastTestHasTypeMatch.get()); + +// // exclude setup, teardown, first test, and last test +// for (int i = 3; i <= tests.size() - 2; i++) { +// ObjectNode otherTest = tests.get(i); +// JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); +// assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); +// ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; +// otherTestParentArray.elements().forEachRemaining(node -> { +// assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); +// ObjectNode childObject = (ObjectNode) node; +// JsonNode matchObject = childObject.get("match"); +// if (matchObject != null) { +// if (matchObject.get("_type") != null) { +// if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { +// assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); +// } +// } +// } +// }); +// } + } + + @Override + protected boolean getHumanDebug() { + return true; + } +} diff --git a/buildSrc/src/test/resources/rest/transform/match/is_true.yml b/buildSrc/src/test/resources/rest/transform/match/is_true.yml new file mode 100644 index 0000000000000..a23d8ad0349bd --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/is_true.yml @@ -0,0 +1,110 @@ +--- +setup: + - do: + something: + here: ok +--- +teardown: + - do: + something_else: + here: true +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { copied.from.real.test.total: 1 } + - match: { hits.hits.0._index: "single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _shards.failed: 0 } + + - do: + and: again + + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _below_is_target_for_tests: 0 } + - is_true: "test_index.mappings.type_1" + +--- +"Also has _type match": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - is_true: "test_index.mappings.type_1" + - match: { _below_is_target_for_tests: 0 } + - match: { _type: foo } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + not_random_but_representive: "of actual test" + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + +--- +"Does not have _type match ": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + it: again + + - match: { _type: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } +--- +"Last test": + + - do: + something: 中文 + + - match: { _index: test_1 } + - is_true: "test_index.mappings.type_1" + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + - do: + something_else: 中文 + + - match: { _index: test_1 } + - match: { _type: "the value does not matter" } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index e8101d069f359..a4e86eff1a95e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -143,9 +143,8 @@ tasks.named("yamlRestCompatTest").configure { 'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types', 'field_caps/10_basic/Get date_nanos field caps', 'field_caps/30_filter/Field caps with index filter', - 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types',// failing due to include_type_name #48632 - 'get/21_stored_fields_with_types/Stored fields', // failing due to include_type_name #48632 - 'get/71_source_filtering_with_types/Source filtering',// failing due to include_type_name #48632 + // WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored) + 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types', 'get_source/11_basic_with_types/Basic with types', 'get_source/16_default_values_with_types/Default values', 'get_source/41_routing_with_types/Routing', @@ -155,8 +154,6 @@ tasks.named("yamlRestCompatTest").configure { 'get_source/81_missing_with_types/Missing document with ignore', 'get_source/86_source_missing_with_types/Missing document source with catch', 'get_source/86_source_missing_with_types/Missing document source with ignore', - 'index/70_mix_typeless_typeful/Index call that introduces new field mappings', // failing due to include_type_name #48632 - 'index/70_mix_typeless_typeful/Index with typeless API on an index that has types', // failing due to include_type_name #48632 'indices.clone/10_basic/Clone index via API', 'indices.create/10_basic/Create index with explicit _doc type', 'indices.create/10_basic/Create index without soft deletes', @@ -173,8 +170,8 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', - 'indices.get/11_basic_with_types/Test include_type_name', - 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', +// 'indices.get/11_basic_with_types/Test include_type_name', +// 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', @@ -355,4 +352,5 @@ tasks.named("yamlRestCompatTest").configure { tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") + task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") }) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index cc9aaf74ef970..f4c996885b00c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -26,6 +27,9 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; + /** * A response for a get index action. */ @@ -192,7 +196,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexMappings == null) { builder.startObject("mappings").endObject(); } else { - builder.field("mappings", indexMappings.sourceAsMap()); + boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + builder.startObject("mappings"); + builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap()); + builder.endObject(); + } else { + builder.field("mappings", indexMappings.sourceAsMap()); + } } builder.startObject("settings"); @@ -221,6 +233,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private void addMappings(XContentBuilder builder, MappingMetadata indexMappings) throws IOException { + if (indexMappings == null) { + builder.startObject("mappings").endObject(); + } else { + builder.field("mappings", indexMappings.sourceAsMap()); + } + } + @Override public String toString() { return Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index c7ed7ebaad560..c699a194286e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; + public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 4bd7272b3c440..59d4db3dc5a38 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -40,6 +41,13 @@ */ public abstract class BaseRestHandler implements RestHandler { + /** + * Parameter that controls whether certain REST apis should include type names in their requests or responses. + * Note: This parameter is only available through compatible rest api. + */ + public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; + public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; + public static final Setting MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope); @@ -72,14 +80,14 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // use a sorted set so the unconsumed parameters appear in a reliable sorted order final SortedSet unconsumedParams = request.unconsumedParams() .stream() - .filter(p -> responseParams().contains(p) == false) + .filter(p -> responseParams(request.getRestApiVersion()).contains(p) == false) .collect(Collectors.toCollection(TreeSet::new)); // validate the non-response params if (unconsumedParams.isEmpty() == false) { final Set candidateParams = new HashSet<>(); candidateParams.addAll(request.consumedParams()); - candidateParams.addAll(responseParams()); + candidateParams.addAll(responseParams(request.getRestApiVersion())); throw new IllegalArgumentException(unrecognized(request, unconsumedParams, candidateParams, "parameter")); } @@ -174,6 +182,10 @@ protected Set responseParams() { return Collections.emptySet(); } + protected Set responseParams(RestApiVersion restApiVersion) { + return responseParams(); + } + public static class Wrapper extends BaseRestHandler { protected final BaseRestHandler delegate; @@ -202,6 +214,11 @@ protected Set responseParams() { return delegate.responseParams(); } + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + return delegate.responseParams(restApiVersion); + } + @Override public boolean canTripCircuitBreaker() { return delegate.canTripCircuitBreaker(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 78801211488f9..621cdc18b1094 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -11,6 +11,9 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; @@ -19,6 +22,7 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,6 +32,10 @@ public class RestCreateIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCreateIndexAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " + + "index requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of(new Route(PUT, "/{index}")); @@ -40,7 +48,60 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + CreateIndexRequest createIndexRequest; + if (request.getRestApiVersion() == RestApiVersion.V_7) { + createIndexRequest = prepareRequestV7(request); + } else { + createIndexRequest = prepareRequest(request); + } + return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); + } + // default scope for testing types in mapping + CreateIndexRequest prepareRequestV7(RestRequest request) { + CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("create_index_with_types", TYPES_DEPRECATION_MESSAGE); + } + + if (request.hasContent()) { + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + + + request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc + sourceAsMap = prepareMappingsV7(sourceAsMap, request); + + createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } + + createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); + createIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createIndexRequest.masterNodeTimeout())); + createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + return createIndexRequest; + } + + static Map prepareMappingsV7(Map source, RestRequest request) { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false); + + @SuppressWarnings("unchecked") + Map mappings = (Map) source.get("mappings"); + + if (includeTypeName && mappings.size() == 1) { + Map newSource = new HashMap<>(); + + String typeName = mappings.keySet().iterator().next(); + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + + // no matter what the type was, replace it with _doc, because the internal representation still uses single type `_doc`. + newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings)); + return newSource; + } else { + return prepareMappings(source); + } + } + + CreateIndexRequest prepareRequest(RestRequest request) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { @@ -53,9 +114,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); createIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createIndexRequest.masterNodeTimeout())); createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); - } + return createIndexRequest; + } static Map prepareMappings(Map source) { if (source.containsKey("mappings") == false diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java index 6394666bd98e0..0b13c6620198c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java @@ -12,15 +12,20 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; @@ -29,6 +34,13 @@ * The REST handler for get index and head index APIs. */ public class RestGetIndicesAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndicesAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests" + + " is deprecated. The parameter will be removed in the next major version."; + + private static final Set COMPATIBLE_RESPONSE_PARAMS = Collections + .unmodifiableSet(Stream.concat(Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER).stream(), Settings.FORMAT_PARAMS.stream()) + .collect(Collectors.toSet())); @Override public List routes() { @@ -44,6 +56,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + // starting with 7.0 we don't include types by default in the response to GET requests + if (request.getRestApiVersion() == RestApiVersion.V_7 && + request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) && request.method().equals(GET)) { + deprecationLogger.compatibleApiWarning("get_indices_with_types", TYPES_DEPRECATION_MESSAGE); + } + String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetIndexRequest getIndexRequest = new GetIndexRequest(); getIndexRequest.indices(indices); @@ -63,4 +81,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC protected Set responseParams() { return Settings.FORMAT_PARAMS; } + + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + if(restApiVersion == RestApiVersion.V_7){ + return COMPATIBLE_RESPONSE_PARAMS; + } else { + return responseParams(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java index fe89d90405777..b38c5d69b6fb8 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java @@ -8,15 +8,29 @@ package org.elasticsearch.rest.action.admin.indices; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + public class RestCreateIndexActionTests extends ESTestCase { public void testPrepareTypelessRequest() throws IOException { @@ -69,4 +83,59 @@ public void testMalformedMappings() throws IOException { Map source = RestCreateIndexAction.prepareMappings(contentAsMap); assertEquals(contentAsMap, source); } + + public void testIncludeTypeName() throws IOException { + RestCreateIndexAction action = new RestCreateIndexAction(); + List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Accept", compatibleMediaType)) + .withMethod(RestRequest.Method.PUT) + .withPath("/some_index") + .withParams(params) + .build(); + + action.prepareRequest(deprecatedRequest, mock(NodeClient.class)); + assertWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); + + RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withPath("/some_index") + .build(); + action.prepareRequest(validRequest, mock(NodeClient.class)); + } + + public void testTypeInMapping() throws IOException { + RestCreateIndexAction action = new RestCreateIndexAction(); + + List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); + + String content = "{\n" + + " \"mappings\": {\n" + + " \"some_type\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Map params = new HashMap<>(); + params.put(RestCreateIndexAction.INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withPath("/some_index") + .withParams(params) + .withContent(new BytesArray(content), null) + .build(); + + CreateIndexRequest createIndexRequest = action.prepareRequestV7(request); + // some_type is replaced with _doc + assertThat(createIndexRequest.mappings(), equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"type\":\"text\"}}}}")); + assertWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 0d654136b23b1..81a23048c9790 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1144,6 +1144,10 @@ public static Class getTestTransportPlugin() { public String randomCompatibleMediaType(RestApiVersion version) { XContentType type = randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML); + return compatibleMediaType(type, version); + } + + public String compatibleMediaType(XContentType type, RestApiVersion version) { return type.toParsedMediaType() .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version.major))); } From 51ef24abdfea63bce5fd1d17bed301a425a0dd98 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 10:54:45 +0200 Subject: [PATCH 02/32] style --- .../rest/transform/RestTestTransformer.java | 7 ++- .../rest/transform/match/ReplaceTextual.java | 2 +- .../transform/match/ReplaceTextualTests.java | 46 +++++++++---------- .../mapping/get/GetMappingsResponse.java | 4 -- .../admin/indices/RestCreateIndexAction.java | 1 - 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java index 2003397acd8af..9b6ef3f5eaa24 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java @@ -144,13 +144,12 @@ private void traverseTest( } else { if (entry.getValue().isObject()) { ObjectNode child = (ObjectNode) entry.getValue(); -// if (child.has(transform.requiredChildKey())) { - if(transform.matches(child)) { + if (child.has(transform.requiredChildKey())) { transform.transformTest((ObjectNode) currentNode); } - } else if(entry.getValue().isTextual()){ + } else if (entry.getValue().isTextual()) { String fieldValue = entry.getValue().asText(); - if(transform.matches(fieldValue)) {///fieldValue.equals(transform.requiredChildKey())){ + if (fieldValue.equals(transform.requiredChildKey())) { transform.transformTest((ObjectNode) currentNode); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java index 86a06d4df9158..a6182dc0bee9d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -54,7 +54,7 @@ public boolean shouldApply(RestTestContext testContext) { @Override public void transformTest(ObjectNode matchParent) { -// ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + // ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); matchParent.set(getKeyToFind(), replacementNode); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 07aa62adc82dc..10b1fa5a0dc32 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -58,9 +58,9 @@ private void validateTest(List tests, boolean beforeTransformation, JsonNode matchObject = childObject.get("is_true"); if (matchObject != null) { firstTestHasMatchObject.set(true); - if (firstTestHasTypeMatch.get() == false ) { + if (firstTestHasTypeMatch.get() == false) { firstTestHasTypeMatch.set(true); - }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); } @@ -82,9 +82,9 @@ private void validateTest(List tests, boolean beforeTransformation, JsonNode matchObject = childObject.get("is_true"); if (matchObject != null) { lastTestHasMatchObject.set(true); - if (lastTestHasTypeMatch.get() == false ) { + if (lastTestHasTypeMatch.get() == false) { lastTestHasTypeMatch.set(true); - }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); } @@ -93,25 +93,25 @@ private void validateTest(List tests, boolean beforeTransformation, assertTrue(lastTestHasMatchObject.get()); assertTrue(lastTestHasTypeMatch.get()); -// // exclude setup, teardown, first test, and last test -// for (int i = 3; i <= tests.size() - 2; i++) { -// ObjectNode otherTest = tests.get(i); -// JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); -// assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); -// ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; -// otherTestParentArray.elements().forEachRemaining(node -> { -// assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); -// ObjectNode childObject = (ObjectNode) node; -// JsonNode matchObject = childObject.get("match"); -// if (matchObject != null) { -// if (matchObject.get("_type") != null) { -// if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { -// assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); -// } -// } -// } -// }); -// } + // // exclude setup, teardown, first test, and last test + // for (int i = 3; i <= tests.size() - 2; i++) { + // ObjectNode otherTest = tests.get(i); + // JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); + // assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + // ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; + // otherTestParentArray.elements().forEachRemaining(node -> { + // assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + // ObjectNode childObject = (ObjectNode) node; + // JsonNode matchObject = childObject.get("match"); + // if (matchObject != null) { + // if (matchObject.get("_type") != null) { + // if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { + // assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); + // } + // } + // } + // }); + // } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index c699a194286e7..c7ed7ebaad560 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -24,9 +23,6 @@ import java.io.IOException; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; - public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 621cdc18b1094..891af2db58287 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; -import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; From 1bbc802c71fcdde1bdd3db69c173e02603c90289 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 12:17:21 +0200 Subject: [PATCH 03/32] test fix --- .../rest/compat/RestCompatTestTransformTask.java | 3 ++- .../transform/RestTestTransformByParentObject.java | 5 +++++ .../test/rest/transform/RestTestTransformer.java | 7 ++++--- .../test/rest/transform/match/ReplaceTextual.java | 14 ++++++++++---- .../rest/transform/match/ReplaceTextualTests.java | 3 ++- .../admin/indices/RestCreateIndexAction.java | 4 ++++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index ab7bf3ae218c2..d7e38ec01353e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; import org.elasticsearch.gradle.Version; @@ -103,7 +104,7 @@ public void replaceMatch(String subKey, Object value) { } public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, JsonNode.class))); + transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, TextNode.class))); } /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java index 531530654f0c5..219630a62880f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java @@ -8,6 +8,7 @@ package org.elasticsearch.gradle.test.rest.transform; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; /** @@ -27,4 +28,8 @@ public interface RestTestTransformByParentObject extends RestTestTransform tests = getTests(testName); - JsonNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", JsonNode.class); + TextNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", TextNode.class); validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 891af2db58287..04437b0ff3137 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; @@ -89,6 +90,9 @@ static Map prepareMappingsV7(Map source, RestReq Map newSource = new HashMap<>(); String typeName = mappings.keySet().iterator().next(); + if (Strings.hasText(typeName) == false) { + throw new IllegalArgumentException("name cannot be empty string"); + } @SuppressWarnings("unchecked") Map typedMappings = (Map) mappings.get(typeName); From 9592c74f88679e29fa1ccc6148d974984d77dc02 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 14:57:40 +0200 Subject: [PATCH 04/32] is true as a parameter --- .../compat/RestCompatTestTransformTask.java | 2 +- .../rest/transform/match/ReplaceTextual.java | 23 +++--- .../transform/match/ReplaceTextualTests.java | 76 ++++++++----------- .../match/{is_true.yml => text_replace.yml} | 6 +- 4 files changed, 49 insertions(+), 58 deletions(-) rename buildSrc/src/test/resources/rest/transform/match/{is_true.yml => text_replace.yml} (94%) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index d7e38ec01353e..9ed6554d26263 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -104,7 +104,7 @@ public void replaceMatch(String subKey, Object value) { } public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, TextNode.class))); + transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); } /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java index 0432805f8b174..ad32bc3bb86c5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -18,21 +18,24 @@ import org.gradle.api.tasks.Optional; /** - * A transformation to replace the flat fields + * A transformation to replace the flat textual fields. */ public class ReplaceTextual implements RestTestTransformByParentObject { - private final String replaceKey; + private final String keyToReplaceName; + private final String valueToBeReplaced; private final TextNode replacementNode; private final String testName; - public ReplaceTextual(String replaceKey, TextNode replacementNode) { - this.replaceKey = replaceKey; + public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; this.replacementNode = replacementNode; this.testName = null; } - public ReplaceTextual(String replaceKey, TextNode replacementNode, String testName) { - this.replaceKey = replaceKey; + public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode, String testName) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; this.replacementNode = replacementNode; this.testName = testName; } @@ -40,12 +43,12 @@ public ReplaceTextual(String replaceKey, TextNode replacementNode, String testNa @Override @Internal public String getKeyToFind() { - return "is_true"; + return keyToReplaceName; } @Override public String requiredChildKey() { - return replaceKey; + return valueToBeReplaced; } @Override @@ -59,8 +62,8 @@ public void transformTest(ObjectNode matchParent) { } @Input - public String getReplaceKey() { - return replaceKey; + public String getValueToBeReplaced() { + return valueToBeReplaced; } @Input diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 11e6dd7cffaa9..7e6788cb75eac 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -30,13 +30,13 @@ public class ReplaceTextualTests extends TransformTests { @Test public void testReplaceAll() throws Exception { - String testName = "/rest/transform/match/is_true.yml"; + String testName = "/rest/transform/match/text_replace.yml"; List tests = getTests(testName); - TextNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", TextNode.class); + TextNode replacementNode = MAPPER.convertValue("_replaced_value", TextNode.class); validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("test_index.mappings.type_1", replacementNode, null)) + Collections.singletonList(new ReplaceTextual("key_to_replace", "key_to_replace", replacementNode, null)) ); printTest(testName, transformedTests); validateTest(tests, false, true); @@ -50,69 +50,57 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; - AtomicBoolean firstTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean firstTestHasTypeMatch = new AtomicBoolean(false); + AtomicBoolean firstTestOccurrenceFound = new AtomicBoolean(false); firstTestParentArray.elements().forEachRemaining(node -> { assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("is_true"); + JsonNode matchObject = childObject.get("key_to_replace"); if (matchObject != null) { - firstTestHasMatchObject.set(true); - if (firstTestHasTypeMatch.get() == false) { - firstTestHasTypeMatch.set(true); - } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ - if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + firstTestOccurrenceFound.set(true); + if (beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); } } }); - assertTrue(firstTestHasMatchObject.get()); - assertTrue(firstTestHasTypeMatch.get()); + assertTrue(firstTestOccurrenceFound.get()); // last test JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; - AtomicBoolean lastTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean lastTestHasTypeMatch = new AtomicBoolean(false); + AtomicBoolean lastTestOccurrenceFound = new AtomicBoolean(false); lastTestParentArray.elements().forEachRemaining(node -> { assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("is_true"); + JsonNode matchObject = childObject.get("key_to_replace"); if (matchObject != null) { - lastTestHasMatchObject.set(true); - if (lastTestHasTypeMatch.get() == false) { - lastTestHasTypeMatch.set(true); - } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ - if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + lastTestOccurrenceFound.set(true); + if (beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); } } }); - assertTrue(lastTestHasMatchObject.get()); - assertTrue(lastTestHasTypeMatch.get()); + assertTrue(lastTestOccurrenceFound.get()); - // // exclude setup, teardown, first test, and last test - // for (int i = 3; i <= tests.size() - 2; i++) { - // ObjectNode otherTest = tests.get(i); - // JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); - // assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - // ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; - // otherTestParentArray.elements().forEachRemaining(node -> { - // assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - // ObjectNode childObject = (ObjectNode) node; - // JsonNode matchObject = childObject.get("match"); - // if (matchObject != null) { - // if (matchObject.get("_type") != null) { - // if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { - // assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); - // } - // } - // } - // }); - // } + // exclude setup, teardown, first test, and last test + for (int i = 3; i <= tests.size() - 2; i++) { + ObjectNode otherTest = tests.get(i); + JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); + assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; + otherTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("key_to_replace"); + if (matchObject != null) { + if (beforeTransformation == false && allTests) { + assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); + } + } + }); + } } @Override diff --git a/buildSrc/src/test/resources/rest/transform/match/is_true.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml similarity index 94% rename from buildSrc/src/test/resources/rest/transform/match/is_true.yml rename to buildSrc/src/test/resources/rest/transform/match/text_replace.yml index a23d8ad0349bd..3016d113d7220 100644 --- a/buildSrc/src/test/resources/rest/transform/match/is_true.yml +++ b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml @@ -34,7 +34,7 @@ teardown: - match: { _shards.successful: 2 } - match: { _shards.skipped : 0} - match: { _below_is_target_for_tests: 0 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" --- "Also has _type match": @@ -47,7 +47,7 @@ teardown: and: again - match: { hits.total.value: 0 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" - match: { _below_is_target_for_tests: 0 } - match: { _type: foo } - match: { _shards.total: 2 } @@ -95,7 +95,7 @@ teardown: something: 中文 - match: { _index: test_1 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" - match: { _id: 中文 } - match: { _source: { foo: "Hello: 中文" } } From b6abb3f67f5ee4c7e6081b75e205a65c32c52c4e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 17:47:15 +0200 Subject: [PATCH 05/32] create index passing except for get mappings api --- .../compat/RestCompatTestTransformTask.java | 3 +++ rest-api-spec/build.gradle | 12 ++++------- .../admin/indices/RestCreateIndexAction.java | 10 ++++----- .../admin/indices/RestGetMappingAction.java | 9 ++++++++ .../indices/RestPutIndexTemplateAction.java | 21 +++++++++++++++++-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 9ed6554d26263..89fedbad549c0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -107,6 +107,9 @@ public void replaceIsTrue(String subKey, Object value) { transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); } + public void replaceIsFalse(String subKey, Object value) { + transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); + } /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a4e86eff1a95e..8206c6bd2646e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -155,23 +155,17 @@ tasks.named("yamlRestCompatTest").configure { 'get_source/86_source_missing_with_types/Missing document source with catch', 'get_source/86_source_missing_with_types/Missing document source with ignore', 'indices.clone/10_basic/Clone index via API', - 'indices.create/10_basic/Create index with explicit _doc type', 'indices.create/10_basic/Create index without soft deletes', - 'indices.create/11_basic_with_types/Create index', - 'indices.create/11_basic_with_types/Create index with aliases', + // 5 below await retrofitting Removes typed URLs from mapping APIs #41676 'indices.create/11_basic_with_types/Create index with mappings', - 'indices.create/11_basic_with_types/Create index with settings', - 'indices.create/11_basic_with_types/Create index with wait_for_active_shards set to all', - 'indices.create/11_basic_with_types/Create index with write aliases', 'indices.create/20_mix_typeless_typeful/Create a typed index while there is a typeless template', 'indices.create/20_mix_typeless_typeful/Create a typeless index while there is a typed template', 'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template', 'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template', + // 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', -// 'indices.get/11_basic_with_types/Test include_type_name', -// 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', @@ -353,4 +347,6 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") + task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") + task.replaceIsFalse("test-1.mappings.my_type", "test-1.mappings._doc") }) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 04437b0ff3137..d5be27fca0d6d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -61,14 +61,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC CreateIndexRequest prepareRequestV7(RestRequest request) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc deprecationLogger.compatibleApiWarning("create_index_with_types", TYPES_DEPRECATION_MESSAGE); } if (request.hasContent()) { Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - - request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc sourceAsMap = prepareMappingsV7(sourceAsMap, request); createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); @@ -86,9 +85,9 @@ static Map prepareMappingsV7(Map source, RestReq @SuppressWarnings("unchecked") Map mappings = (Map) source.get("mappings"); - if (includeTypeName && mappings.size() == 1) { + if (includeTypeName && mappings != null && mappings.size() == 1) { Map newSource = new HashMap<>(); - + newSource.putAll(source); // mappings will be overridden. Aliases, settings stay the same String typeName = mappings.keySet().iterator().next(); if (Strings.hasText(typeName) == false) { throw new IllegalArgumentException("name cannot be empty string"); @@ -132,7 +131,8 @@ static Map prepareMappings(Map source) { @SuppressWarnings("unchecked") Map mappings = (Map) source.get("mappings"); if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); } newSource.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 53585818546f4..a6d271991a2bc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; @@ -32,6 +33,9 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestGetMappingAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get" + + " mapping requests is deprecated. The parameter will be removed in the next major version."; private final ThreadPool threadPool; @@ -55,6 +59,11 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.compatibleApiWarning("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 05b8bfc998cf0..12ab2346e2107 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -25,7 +27,10 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestPutIndexTemplateAction extends BaseRestHandler { - + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutIndexTemplateAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in put index template requests is deprecated."+ + " The parameter will be removed in the next major version."; @Override public List routes() { return List.of( @@ -50,7 +55,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + if(request.getRestApiVersion() == RestApiVersion.V_7) { + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) ) { + deprecationLogger.compatibleApiWarning("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); + } + if(includeTypeName) { + sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request); + } else { + sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + } + } else { + sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + } putRequest.source(sourceAsMap); return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel)); From 4ead4002c9eb38b6d3d79a32f2e7f1c759b2b64c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 09:31:53 +0200 Subject: [PATCH 06/32] code style --- .../gradle/internal/rest/compat/RestCompatTestTransformTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 89fedbad549c0..3f5247e244c6f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -110,6 +110,7 @@ public void replaceIsTrue(String subKey, Object value) { public void replaceIsFalse(String subKey, Object value) { transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); } + /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * From adead4d082b0d453d64f4b0fa2e53a8f47b1640e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 10:10:33 +0200 Subject: [PATCH 07/32] test fix --- .../test/rest/transform/match/ReplaceTextualTests.java | 3 +-- .../test/resources/rest/transform/match/text_replace.yml | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 7e6788cb75eac..fd710b0bc497e 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -36,11 +36,10 @@ public void testReplaceAll() throws Exception { validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("key_to_replace", "key_to_replace", replacementNode, null)) + Collections.singletonList(new ReplaceTextual("key_to_replace", "value_to_replace", replacementNode, null)) ); printTest(testName, transformedTests); validateTest(tests, false, true); - } private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { diff --git a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml index 3016d113d7220..a3f71bc33ffd0 100644 --- a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml +++ b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml @@ -34,7 +34,7 @@ teardown: - match: { _shards.successful: 2 } - match: { _shards.skipped : 0} - match: { _below_is_target_for_tests: 0 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" --- "Also has _type match": @@ -47,7 +47,7 @@ teardown: and: again - match: { hits.total.value: 0 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" - match: { _below_is_target_for_tests: 0 } - match: { _type: foo } - match: { _shards.total: 2 } @@ -95,7 +95,7 @@ teardown: something: 中文 - match: { _index: test_1 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" - match: { _id: 中文 } - match: { _source: { foo: "Hello: 中文" } } From c8f7413e25ebb7e0401b5ff4f479ea5d74d5c2dd Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 13:42:55 +0200 Subject: [PATCH 08/32] fix test --- .../test/rest/transform/match/ReplaceTextualTests.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index fd710b0bc497e..ac6ed772b5576 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -93,10 +93,9 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - if (beforeTransformation == false && allTests) { - assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); - } + if (matchObject != null && matchObject.get("key_to_replace") != null + && beforeTransformation == false && allTests) { + assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); } }); } From 57e63fcd2dffb56e9866950d100679304da26186 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 14:12:34 +0200 Subject: [PATCH 09/32] style check --- .../gradle/test/rest/transform/match/ReplaceTextualTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index ac6ed772b5576..0f05a4ba2cc7d 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -93,8 +93,7 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null && matchObject.get("key_to_replace") != null - && beforeTransformation == false && allTests) { + if (matchObject != null && matchObject.get("key_to_replace") != null && beforeTransformation == false && allTests) { assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); } }); From 539b4217969902964db67b9a11b705b4d78c9d3f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 17:01:43 +0200 Subject: [PATCH 10/32] add all changes --- rest-api-spec/build.gradle | 51 ++++++++----------- .../admin/indices/get/GetIndexResponse.java | 8 --- .../mapping/get/GetFieldMappingsResponse.java | 16 +++++- .../indices/rollover/RolloverRequest.java | 26 ++++++++-- .../get/GetIndexTemplatesResponse.java | 15 +++++- .../metadata/IndexTemplateMetadata.java | 5 +- .../elasticsearch/rest/BaseRestHandler.java | 8 +++ .../indices/RestGetFieldMappingAction.java | 21 ++++++++ .../indices/RestGetIndexTemplateAction.java | 23 +++++++++ .../indices/RestRolloverIndexAction.java | 22 +++++++- .../rollover/RolloverRequestTests.java | 8 +-- x-pack/plugin/build.gradle | 13 +++++ 12 files changed, 164 insertions(+), 52 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index b6c14a05cc1f9..40538c28a2ff4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -145,7 +145,6 @@ tasks.named("yamlRestCompatTest").configure { 'explain/21_source_filtering_with_types/Source filtering', 'explain/31_query_string_with_types/explain with query_string parameters', 'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types', - 'field_caps/10_basic/Get date_nanos field caps', 'field_caps/30_filter/Field caps with index filter', // WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored) 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types', @@ -170,23 +169,23 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', - 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type', - 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do', - 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names', - 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types', + 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type',// awaits #41676 + 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',// awaits #41676 + 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names',// awaits #41676 + 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types',// awaits #41676 'indices.get_mapping/11_basic_with_types/Get /*/_mapping/{type}', 'indices.get_mapping/11_basic_with_types/Get /_all/_mapping/{type}', 'indices.get_mapping/11_basic_with_types/Get /_mapping', @@ -207,8 +206,8 @@ tasks.named("yamlRestCompatTest").configure { 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', - 'indices.get_template/11_basic_with_types/Get template', - 'indices.get_template/11_basic_with_types/Get template with no mappings', +// 'indices.get_template/11_basic_with_types/Get template', +// 'indices.get_template/11_basic_with_types/Get template with no mappings', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', @@ -223,15 +222,9 @@ tasks.named("yamlRestCompatTest").configure { 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', 'indices.put_mapping/all_path_options_with_types/put one mapping per index', - 'indices.put_template/10_basic/Put template with explicit _doc type', - 'indices.put_template/11_basic_with_types/Put multiple template', - 'indices.put_template/11_basic_with_types/Put template', + // needs discussion about IndexTemplateMetadata#toInnerXContent and reducingMappings when includeTypeName and empty mappings etc 'indices.put_template/11_basic_with_types/Put template with empty mappings', - 'indices.rollover/10_basic/Rollover index via API', - 'indices.rollover/20_max_doc_condition/Max docs rollover conditions matches only primary shards', - 'indices.rollover/30_max_size_condition/Rollover with max_size condition', - 'indices.rollover/40_mapping/Mappings with explicit _doc type', - 'indices.rollover/41_mapping_with_types/Typeless mapping', + // 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', 'indices.shard_stores/10_basic/basic index test', @@ -292,8 +285,6 @@ tasks.named("yamlRestCompatTest").configure { 'search.aggregation/10_histogram/Deprecated _time order', 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', 'search.aggregation/20_terms/Deprecated _term order', - 'search.aggregation/280_geohash_grid/Basic test', - 'search.aggregation/290_geotile_grid/Basic test', 'search.aggregation/51_filter_with_types/Filter aggs with terms lookup and ensure it\'s cached', 'search.inner_hits/10_basic/Nested doc version and seqIDs', 'search.inner_hits/10_basic/Nested inner hits', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index f4c996885b00c..b6d9beb29b4ba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -233,14 +233,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private void addMappings(XContentBuilder builder, MappingMetadata indexMappings) throws IOException { - if (indexMappings == null) { - builder.startObject("mappings").endObject(); - } else { - builder.field("mappings", indexMappings.sourceAsMap()); - } - } - @Override public String toString() { return Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index e91492c1344e1..254dbed30c26f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -22,6 +23,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; import java.io.IOException; import java.io.InputStream; @@ -30,6 +32,7 @@ import java.util.Objects; import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; /** * Response object for {@link GetFieldMappingsRequest} API @@ -84,15 +87,24 @@ public FieldMappingMetadata fieldMappings(String index, String field) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean includeTypeName = params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + builder.startObject(); for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); - if (indexEntry.getValue() != null) { - addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + builder.startObject(MapperService.SINGLE_MAPPING_NAME); + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + builder.endObject(); + } else { + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + } } + builder.endObject(); builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 248c8c162d16c..44626a6af41bd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; @@ -37,7 +38,7 @@ */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser>, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -67,6 +68,22 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); + PARSER.declareField((parser, request, includeTypeName) -> { + if (includeTypeName) { + //expecting one type only + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping((Map) mappingsEntry.getValue()); + } + } else { + // a type is not included, add a dummy _doc type + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); + } + request.createIndexRequest.mapping(mappings); + } + }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type Map mappings = parser.map(); @@ -74,7 +91,8 @@ public class RolloverRequest extends AcknowledgedRequest implem throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); } request.createIndexRequest.mapping(mappings); - }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); + }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); + PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), CreateIndexRequest.ALIASES, ObjectParser.ValueType.OBJECT); } @@ -238,7 +256,7 @@ public CreateIndexRequest getCreateIndexRequest() { } // param isTypeIncluded decides how mappings should be parsed from XContent - public void fromXContent(XContentParser parser) throws IOException { - PARSER.parse(parser, this, null); + public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { + PARSER.parse(parser, this, isTypeIncluded); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9b95c146b79a7..f3332b734080f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -20,6 +21,8 @@ import java.util.Objects; import static java.util.Collections.singletonMap; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject { @@ -63,9 +66,19 @@ public int hashCode() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params); + boolean includeTypeName = false; + if(builder.getRestApiVersion() == RestApiVersion.V_7) { + includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + } + builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { - IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); + if (includeTypeName) { + IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); + } else { + IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); + } } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java index 855e357bb6f93..ef592c48b2a14 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; @@ -378,7 +379,9 @@ private static void toInnerXContent(IndexTemplateMetadata indexTemplateMetadata, indexTemplateMetadata.settings().toXContent(builder, params); builder.endObject(); - includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); + if(builder.getRestApiVersion() != RestApiVersion.V_7) { + includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); + } CompressedXContent m = indexTemplateMetadata.mappings(); if (m != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 59d4db3dc5a38..2d2d0cc83a0d3 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -182,6 +182,14 @@ protected Set responseParams() { return Collections.emptySet(); } + /** + * Parameters used for controlling the response and thus might not be consumed during + * preparation of the request execution. The value depends on the RestApiVersion provided + * by a user on a request. + * Used in RestHandlers with Compatible Rest Api + * @param restApiVersion - a version provided by a user on a request + * @return a set of parameters used to control the response, depending on a restApiVersion + */ protected Set responseParams(RestApiVersion restApiVersion) { return responseParams(); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 4e192bb996ffa..25283a235ef33 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -8,12 +8,16 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; @@ -32,6 +36,11 @@ public class RestGetFieldMappingAction extends BaseRestHandler { + private static final Logger logger = LogManager.getLogger(RestGetFieldMappingAction.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + "field mapping requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of( @@ -49,6 +58,18 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); + if(request.getRestApiVersion() == RestApiVersion.V_7){ + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); +// if (includeTypeName == false /*&& types.length > 0*/) { +// throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + +// " is set to true."); +// } + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + } + + GetFieldMappingsRequest getMappingsRequest = new GetFieldMappingsRequest(); getMappingsRequest.indices(indices).fields(fields).includeDefaults(request.paramAsBoolean("include_defaults", false)); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 10b97915f1e7e..fa51cd4bf96f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -11,14 +11,18 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -31,6 +35,11 @@ * The REST handler for get template and head template APIs. */ public class RestGetIndexTemplateAction extends BaseRestHandler { + private static final Set COMPATIBLE_RESPONSE_PARAMS = Collections.unmodifiableSet(Sets.union( + Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER), Settings.FORMAT_PARAMS)); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndexTemplateAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + "index template requests is deprecated. The parameter will be removed in the next major version."; @Override public List routes() { @@ -47,6 +56,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if(request.getRestApiVersion() == RestApiVersion.V_7) { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); + } + } final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); @@ -73,4 +88,12 @@ protected Set responseParams() { return Settings.FORMAT_PARAMS; } + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + if(restApiVersion == RestApiVersion.V_7){ + return COMPATIBLE_RESPONSE_PARAMS; + } else { + return responseParams(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 025e291176ef4..f6f8e450e4838 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -22,6 +24,10 @@ public class RestRolloverIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestRolloverIndexAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in rollover " + + "index requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of( @@ -36,13 +42,25 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = isIncludeTypeName(request); RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(rolloverIndexRequest::fromXContent); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); rolloverIndexRequest.getCreateIndexRequest().waitForActiveShards( - ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + ActiveShardCount.parseString(request.param("wait_for_active_shards"))); return channel -> client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel)); } + + private boolean isIncludeTypeName(RestRequest request) { + boolean includeTypeName = false; + if (request.getRestApiVersion() == RestApiVersion.V_7) { + includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); + } + } + return includeTypeName; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index ced6901b6daaa..71156785b66ca 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -60,7 +60,7 @@ public void testConditionsParsing() throws Exception { .field("max_primary_shard_size", "55gb") .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(4)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -97,7 +97,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings(), containsString("not_analyzed")); @@ -118,7 +118,7 @@ public void testTypelessMappingParsing() throws Exception { .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); CreateIndexRequest createIndexRequest = request.getCreateIndexRequest(); String mapping = createIndexRequest.mappings(); @@ -167,7 +167,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 973d46e6406e1..b09b791c579fa 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,6 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -130,3 +131,15 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } +//tasks.named("yamlRestCompatTest").configure { +// onlyIf { +// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits +// OS.current() != OS.WINDOWS +// } +// systemProperty 'tests.rest.blacklist', [] +//} +// +//tasks.named("transformV7RestTests").configure({ task -> +// task.replaceMatch("_type", "_doc") +// task.addAllowedWarningRegex("\\[types removal\\].*") +//}) From 27c80c04f327bd0e91a1cb8eb42923be59bcc7a6 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 09:53:06 +0200 Subject: [PATCH 11/32] disable compat xpack tests --- x-pack/plugin/build.gradle | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index b09b791c579fa..ffdf1924bdef3 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,7 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' -apply plugin: 'elasticsearch.yaml-rest-compat-test' +//apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -15,6 +15,7 @@ dependencies { // let the yamlRestTest see the classpath of test GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest")) +//GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest", tasks.named("yamlRestCompatTest")) restResources { restApi { @@ -132,11 +133,13 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } //tasks.named("yamlRestCompatTest").configure { -// onlyIf { -// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits -// OS.current() != OS.WINDOWS -// } -// systemProperty 'tests.rest.blacklist', [] +//// onlyIf { +//// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits +//// OS.current() != OS.WINDOWS +//// } +// systemProperty 'tests.rest.blacklist', [ +// +// ] //} // //tasks.named("transformV7RestTests").configure({ task -> From 4c707bb455142554520c367ab67c8ee354771d10 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 12:49:07 +0200 Subject: [PATCH 12/32] test --- .../get/GetFieldMappingsResponseTests.java | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index b967195eea83b..d04a565827892 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -9,18 +9,33 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; + public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase { public void testManualSerialization() throws IOException { @@ -47,6 +62,55 @@ public void testNullFieldMappingToXContent() { assertEquals("{\"index\":{\"mappings\":{}}}", Strings.toString(response)); } + public void testToXContentIncludesType() throws Exception { + Map> mappings = new HashMap<>(); + FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}")); + mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata)); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); + ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, + "true")); + + // v7 with include_type_name attaches _doc + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { + response.toXContent(builder, params); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map>> index = (Map>>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME)); + assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field")); + } + } + + // v7 with no include_type_name do not attach _doc + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + //v8 does not have _doc, een when include_type_name is present + // (although this throws unconsumed parameter exception in RestGetFieldMappings) + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { + response.toXContent(builder, params); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + } + @Override protected GetFieldMappingsResponse createTestInstance() { return new GetFieldMappingsResponse(randomMapping()); @@ -61,7 +125,7 @@ private Map> randomMapping() { Map> mappings = new HashMap<>(); int indices = randomInt(10); - for(int i = 0; i < indices; i++) { + for (int i = 0; i < indices; i++) { Map fieldMappings = new HashMap<>(); int fields = randomInt(10); for (int k = 0; k < fields; k++) { From 4dcd63ca39272620dd44cc4c78b81b79c19a4d0a Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 13:54:39 +0200 Subject: [PATCH 13/32] unit tests --- .../indices/rollover/RolloverRequest.java | 4 +- .../index/mapper/MapperService.java | 5 ++ .../get/GetFieldMappingsResponseTests.java | 16 ++--- .../rollover/RolloverRequestTests.java | 45 ++++++++++++ .../indices/RestGetIndicesActionTests.java | 71 +++++++++++++++++++ .../RestPutIndexTemplateActionTests.java | 69 ++++++++++++++++++ 6 files changed, 198 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 44626a6af41bd..b2d49b85411e0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -77,7 +77,7 @@ public class RolloverRequest extends AcknowledgedRequest implem } else { // a type is not included, add a dummy _doc type Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + if (MapperService.isMappingSourceTyped(mappings)) { throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); } @@ -87,7 +87,7 @@ public class RolloverRequest extends AcknowledgedRequest implem PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + if (MapperService.isMappingSourceTyped(mappings)) { throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); } request.createIndexRequest.mapping(mappings); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 83937f19468ed..6735281819fd1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -333,6 +333,11 @@ public static boolean isMappingSourceTyped(String type, Map mapp return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } + public static boolean isMappingSourceTyped(Map mapping) { + return mapping.size() == 1 && + mapping.keySet().iterator().next().equals("properties") == false; + } + /** * Resolves a type from a mapping-related request into the type that should be used when * merging and updating mappings. diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index d04a565827892..1b235c50e3a0a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; -import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -17,23 +16,19 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase { @@ -75,7 +70,8 @@ public void testToXContentIncludesType() throws Exception { response.toXContent(builder, params); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - Map>> index = (Map>>) parser.map().get("index"); + Map>> index = + (Map>>) parser.map().get("index"); assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME)); assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field")); } @@ -90,8 +86,8 @@ public void testToXContentIncludesType() throws Exception { assertThat(index.get("mappings"), hasKey("field")); } } - //v8 does not have _doc, een when include_type_name is present - // (although this throws unconsumed parameter exception in RestGetFieldMappings) + //v8 does not have _doc, even when include_type_name is present + // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction) try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { response.toXContent(builder, params); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 71156785b66ca..4dc413b5fb834 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -23,7 +24,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; @@ -186,6 +189,48 @@ public void testValidation() { assertEquals("rollover target is missing", validationException.validationErrors().get(0)); } + public void testParsingWithType() throws Exception { + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("conditions") + .field("max_age", "10d") + .field("max_docs", 100) + .endObject() + .startObject("mappings") + .startObject("type1") //TO ASK did we allow types or only _doc? + .startObject("properties") + .startObject("field1") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .startObject("settings") + .field("number_of_shards", 10) + .endObject() + .startObject("aliases") + .startObject("alias1").endObject() + .endObject() + .endObject(); + + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + request.fromXContent(true, parser); + Map> conditions = request.getConditions(); + assertThat(conditions.size(), equalTo(2)); + assertThat(request.getCreateIndexRequest().mappings(), + equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"index\":\"not_analyzed\",\"type\":\"string\"}}}}")); + } + + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); + } + } + private static List> conditionsGenerator = new ArrayList<>(); static { conditionsGenerator.add((request) -> request.addMaxIndexDocsCondition(randomNonNegativeLong())); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java new file mode 100644 index 0000000000000..38e1c6d6cc13d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + +public class RestGetIndicesActionTests extends ESTestCase { + final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + /** + * Test that setting the "include_type_name" parameter raises a warning for the GET request + */ + public void testIncludeTypeNamesWarning() throws IOException { + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index") + .withParams(params) + .build(); + + RestGetIndicesAction handler = new RestGetIndicesAction(); + handler.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE); + + // the same request without the parameter should pass without warning + request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index") + .build(); + handler.prepareRequest(request, mock(NodeClient.class)); + } + + /** + * Test that setting the "include_type_name" parameter doesn't raises a warning if the HEAD method is used (indices.exists) + */ + public void testIncludeTypeNamesWarningExists() throws IOException { + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.HEAD) + .withPath("/some_index") + .withParams(params) + .build(); + + RestGetIndicesAction handler = new RestGetIndicesAction(); + handler.prepareRequest(request, mock(NodeClient.class)); + } +} diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java new file mode 100644 index 0000000000000..1308d5762f8e3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.indices; + + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + +public class RestPutIndexTemplateActionTests extends ESTestCase { + final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + private RestPutIndexTemplateAction action; + + @Before + public void setUpAction() { + action = new RestPutIndexTemplateAction(); + } + + public void testIncludeTypeName() throws IOException { + XContentBuilder typedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.PUT) + .withParams(params) + .withPath("/_template/_some_template") + .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .build(); + action.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); + } +} From b384677fa88da43c180921562e4d3bfe75774a39 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 14:20:41 +0200 Subject: [PATCH 14/32] enable xpack compat tests --- x-pack/plugin/build.gradle | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ffdf1924bdef3..380d0f8320963 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,7 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' -//apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -15,7 +15,7 @@ dependencies { // let the yamlRestTest see the classpath of test GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest")) -//GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest", tasks.named("yamlRestCompatTest")) +GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest") restResources { restApi { @@ -132,17 +132,3 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -//tasks.named("yamlRestCompatTest").configure { -//// onlyIf { -//// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits -//// OS.current() != OS.WINDOWS -//// } -// systemProperty 'tests.rest.blacklist', [ -// -// ] -//} -// -//tasks.named("transformV7RestTests").configure({ task -> -// task.replaceMatch("_type", "_doc") -// task.addAllowedWarningRegex("\\[types removal\\].*") -//}) From 0cc442767002652c955418837c79caeb01b6eb84 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 15:00:08 +0200 Subject: [PATCH 15/32] small cleanups --- rest-api-spec/build.gradle | 2 -- .../action/admin/indices/get/GetIndexResponse.java | 5 ++--- .../indices/mapping/get/GetFieldMappingsResponse.java | 5 ++--- .../indices/template/get/GetIndexTemplatesResponse.java | 9 ++------- .../org/elasticsearch/index/mapper/MapperService.java | 5 ++++- .../action/admin/indices/RestGetFieldMappingAction.java | 6 +----- .../action/admin/indices/RestGetIndexTemplateAction.java | 1 - .../action/admin/indices/RestPutIndexTemplateAction.java | 2 +- .../action/admin/indices/RestRolloverIndexAction.java | 2 +- 9 files changed, 13 insertions(+), 24 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 40538c28a2ff4..b74cd818ae799 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -206,8 +206,6 @@ tasks.named("yamlRestCompatTest").configure { 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', -// 'indices.get_template/11_basic_with_types/Get template', -// 'indices.get_template/11_basic_with_types/Get template with no mappings', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index b6d9beb29b4ba..74fae9eaacfbd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -196,9 +196,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexMappings == null) { builder.startObject("mappings").endObject(); } else { - boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + if (builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { builder.startObject("mappings"); builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 254dbed30c26f..bb1443d9d76f3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -87,15 +87,14 @@ public FieldMappingMetadata fieldMappings(String index, String field) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - boolean includeTypeName = params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); builder.startObject(); for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); if (indexEntry.getValue() != null) { - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + if (builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { builder.startObject(MapperService.SINGLE_MAPPING_NAME); addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index f3332b734080f..f54fc070a3425 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -66,15 +66,10 @@ public int hashCode() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params); - boolean includeTypeName = false; - if(builder.getRestApiVersion() == RestApiVersion.V_7) { - includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); - } - builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { - if (includeTypeName) { + if(builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); } else { IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 6735281819fd1..82d7a49531dd0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -332,7 +332,10 @@ public DocumentMapper documentMapper() { public static boolean isMappingSourceTyped(String type, Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } - + /** + * Returns {@code true} if the given {@code mappingSource} includes any type (anything other than properties) + * as a top-level object. + */ public static boolean isMappingSourceTyped(Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals("properties") == false; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 25283a235ef33..530198a5b7136 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -59,12 +59,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); if(request.getRestApiVersion() == RestApiVersion.V_7){ - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); -// if (includeTypeName == false /*&& types.length > 0*/) { -// throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + -// " is set to true."); -// } if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index fa51cd4bf96f2..b3379899a6f40 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -58,7 +58,6 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { if(request.getRestApiVersion() == RestApiVersion.V_7) { if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.param(INCLUDE_TYPE_NAME_PARAMETER); deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 12ab2346e2107..337aedc7ea696 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -56,10 +56,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); if(request.getRestApiVersion() == RestApiVersion.V_7) { - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) ) { deprecationLogger.compatibleApiWarning("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); } + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if(includeTypeName) { sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request); } else { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index f6f8e450e4838..0c4cf0c9e2f5a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -56,10 +56,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC private boolean isIncludeTypeName(RestRequest request) { boolean includeTypeName = false; if (request.getRestApiVersion() == RestApiVersion.V_7) { - includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { deprecationLogger.compatibleApiWarning("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); } + includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); } return includeTypeName; } From 64c7e90bd2c9d86ea4603fc881e41b4d5e065838 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 15:03:58 +0200 Subject: [PATCH 16/32] fix test --- .../action/admin/indices/RestPutIndexTemplateActionTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java index 1308d5762f8e3..4fc39dde435b8 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -30,7 +30,7 @@ import static org.mockito.Mockito.mock; public class RestPutIndexTemplateActionTests extends ESTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); private RestPutIndexTemplateAction action; @@ -61,7 +61,7 @@ public void testIncludeTypeName() throws IOException { .withMethod(RestRequest.Method.PUT) .withParams(params) .withPath("/_template/_some_template") - .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .withContent(BytesReference.bytes(typedContent), null) .build(); action.prepareRequest(request, mock(NodeClient.class)); assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); From 734e8b8df910372e16e63d23624b5d63e5ffe7cf Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 10:02:07 +0200 Subject: [PATCH 17/32] add blacklist --- x-pack/plugin/build.gradle | 72 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 380d0f8320963..17a481ec32494 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -132,3 +132,75 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } + +tasks.named("yamlRestCompatTest").configure { + + systemProperty 'tests.rest.blacklist', [ + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', + 'sql/sql/Clean cursor', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'sql/sql/Paging through results', + 'privileges/11_builtin/Test get builtin privileges', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'sql/translate/Translate SQL', + 'vectors/15_dense_vector_l1l2/L2 norm', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'rollup/get_jobs/Test basic get_jobs', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/10_dense_vector_basic/Cosine Similarity', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'rollup/rollup_search/Obsolete BWC Timezone', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'rollup/put_job/Test put_job in non-rollup index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'vectors/20_dense_vector_special_cases/Documents missing a vector field', + 'vectors/10_dense_vector_basic/Dot Product', + 'rollup/rollup_search/Obsolete Timezone', + 'rollup/put_job/Test basic put_job', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'sql/sql/Getting textual representation', + 'ml/trained_model_cat_apis/Test cat trained models', + 'rollup/start_job/Test start job twice', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'vectors/50_vector_stats/Usage stats on vector fields', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/15_dense_vector_l1l2/L1 norm', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'rollup/delete_job/Test delete running job', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'rollup/delete_job/Test basic delete_job', + 'license/30_enterprise_license/Installing enterprise license', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'sql/sql/Execute some SQL', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'rollup/delete_job/Test delete job twice', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + ] +} From 276fc88be5705170eb7177ad48d7504cd2e1e10f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 10:05:05 +0200 Subject: [PATCH 18/32] sort and join --- x-pack/plugin/build.gradle | 110 ++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 17a481ec32494..f88be8afa01d9 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -136,71 +136,71 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTest").configure { systemProperty 'tests.rest.blacklist', [ - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'license/30_enterprise_license/Installing enterprise license', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', - 'sql/sql/Clean cursor', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'sql/sql/Paging through results', - 'privileges/11_builtin/Test get builtin privileges', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'sql/translate/Translate SQL', - 'vectors/15_dense_vector_l1l2/L2 norm', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'rollup/get_jobs/Test basic get_jobs', + 'ml/job_cat_apis/Test cat anomaly detector jobs', 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/10_dense_vector_basic/Cosine Similarity', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'vectors/35_sparse_vector_l1l2/L1 norm', + 'ml/trained_model_cat_apis/Test cat trained models', + 'privileges/11_builtin/Test get builtin privileges', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', 'rollup/put_job/Test put_job in non-rollup index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'vectors/20_dense_vector_special_cases/Documents missing a vector field', - 'vectors/10_dense_vector_basic/Dot Product', + 'rollup/rollup_search/Obsolete BWC Timezone', 'rollup/rollup_search/Obsolete Timezone', - 'rollup/put_job/Test basic put_job', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'sql/sql/Getting textual representation', - 'ml/trained_model_cat_apis/Test cat trained models', 'rollup/start_job/Test start job twice', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'vectors/50_vector_stats/Usage stats on vector fields', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/15_dense_vector_l1l2/L1 norm', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'rollup/delete_job/Test delete running job', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'rollup/delete_job/Test basic delete_job', - 'license/30_enterprise_license/Installing enterprise license', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + 'sql/sql/Clean cursor', 'sql/sql/Execute some SQL', + 'sql/sql/Getting textual representation', + 'sql/sql/Paging through results', + 'sql/translate/Translate SQL', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'rollup/delete_job/Test delete job twice', + 'vectors/10_dense_vector_basic/Cosine Similarity', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/10_dense_vector_basic/Dot Product', + 'vectors/15_dense_vector_l1l2/L1 norm', + 'vectors/15_dense_vector_l1l2/L2 norm', + 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', + 'vectors/20_dense_vector_special_cases/Documents missing a vector field', + 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', + 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', 'vectors/35_sparse_vector_l1l2/L2 norm', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' - ] + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields', + ].join(',') } From 07db905c54d9b3f9598825321808694e3af49788 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 11:21:45 +0200 Subject: [PATCH 19/32] fix build.gradle --- x-pack/plugin/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f88be8afa01d9..77b285921b285 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -172,7 +172,7 @@ tasks.named("yamlRestCompatTest").configure { 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', 'sql/sql/Clean cursor', 'sql/sql/Execute some SQL', 'sql/sql/Getting textual representation', From bc0e5df6026c74f934c7aade20c3c0deb2cf766c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 12:34:03 +0200 Subject: [PATCH 20/32] enable all tests --- x-pack/plugin/build.gradle | 147 +++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 77b285921b285..4f263f3de0631 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,74 +133,79 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -tasks.named("yamlRestCompatTest").configure { - - systemProperty 'tests.rest.blacklist', [ - 'license/30_enterprise_license/Installing enterprise license', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/post_data/Test flush with skip_time', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'ml/trained_model_cat_apis/Test cat trained models', - 'privileges/11_builtin/Test get builtin privileges', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'rollup/delete_job/Test basic delete_job', - 'rollup/delete_job/Test delete job twice', - 'rollup/delete_job/Test delete running job', - 'rollup/get_jobs/Test basic get_jobs', - 'rollup/put_job/Test basic put_job', - 'rollup/put_job/Test put_job in non-rollup index', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'rollup/rollup_search/Obsolete Timezone', - 'rollup/start_job/Test start job twice', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', - 'sql/sql/Clean cursor', - 'sql/sql/Execute some SQL', - 'sql/sql/Getting textual representation', - 'sql/sql/Paging through results', - 'sql/translate/Translate SQL', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'vectors/10_dense_vector_basic/Cosine Similarity', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/10_dense_vector_basic/Dot Product', - 'vectors/15_dense_vector_l1l2/L1 norm', - 'vectors/15_dense_vector_l1l2/L2 norm', - 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', - 'vectors/20_dense_vector_special_cases/Documents missing a vector field', - 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', - 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', - 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/35_sparse_vector_l1l2/L1 norm', - 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/50_vector_stats/Usage stats on vector fields', - ].join(',') -} +//tasks.named("yamlRestCompatTest").configure { +// +// systemProperty 'tests.rest.blacklist', [ +// 'license/30_enterprise_license/Installing enterprise license', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', +// 'ml/datafeed_cat_apis/Test cat datafeeds', +// 'ml/datafeeds_crud/Test update datafeed to point to different job', +// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', +// 'ml/datafeeds_crud/Test update datafeed to point to missing job', +// 'ml/job_cat_apis/Test cat anomaly detector jobs', +// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', +// 'ml/jobs_get_stats/Test get job stats for closed job', +// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', +// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', +// 'ml/post_data/Test flush with skip_time', +// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', +// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', +// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', +// 'ml/trained_model_cat_apis/Test cat trained models', +// 'privileges/11_builtin/Test get builtin privileges', +// 'roles/11_idx_arrays/Test put role api using as array of index names', +// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', +// 'rollup/delete_job/Test basic delete_job', +// 'rollup/delete_job/Test delete job twice', +// 'rollup/delete_job/Test delete running job', +// 'rollup/get_jobs/Test basic get_jobs', +// 'rollup/put_job/Test basic put_job', +// 'rollup/put_job/Test put_job in non-rollup index', +// 'rollup/rollup_search/Obsolete BWC Timezone', +// 'rollup/rollup_search/Obsolete Timezone', +// 'rollup/start_job/Test start job twice', +// 'security/authz/14_cat_indices/Test empty request while no-authorized index', +// 'security/authz/14_cat_indices/Test empty request while single authorized index', +// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', +// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', +// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', +// 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', +// 'sql/sql/Clean cursor', +// 'sql/sql/Execute some SQL', +// 'sql/sql/Getting textual representation', +// 'sql/sql/Paging through results', +// 'sql/translate/Translate SQL', +// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', +// 'transform/transforms_cat_apis/Test cat transform stats with column selection', +// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', +// 'vectors/10_dense_vector_basic/Cosine Similarity', +// 'vectors/10_dense_vector_basic/Deprecated function signature', +// 'vectors/10_dense_vector_basic/Dot Product', +// 'vectors/15_dense_vector_l1l2/L1 norm', +// 'vectors/15_dense_vector_l1l2/L2 norm', +//// 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', +//// 'vectors/20_dense_vector_special_cases/Documents missing a vector field', +//// 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', +//// 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', +//// 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', +// 'vectors/30_sparse_vector_basic/Cosine Similarity', +// 'vectors/30_sparse_vector_basic/Deprecated function signature', +// 'vectors/30_sparse_vector_basic/Dot Product', +// 'vectors/35_sparse_vector_l1l2/L1 norm', +// 'vectors/35_sparse_vector_l1l2/L2 norm', +// 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', +// 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', +// 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', +// 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', +// 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', +// 'vectors/50_vector_stats/Usage stats on vector fields', +// ].join(',') +//} + +tasks.named("transformV7RestTests").configure({ task -> + task.replaceMatch("_type", "_doc") + task.addAllowedWarningRegex("\\[types removal\\].*") +}) From 0835b800c8a013550eb4fba58b2f3add2ffbe9f3 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 12:58:54 +0200 Subject: [PATCH 21/32] fix block list --- x-pack/plugin/build.gradle | 132 +++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 4f263f3de0631..c5fb427a1ead4 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,77 +133,67 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -//tasks.named("yamlRestCompatTest").configure { -// -// systemProperty 'tests.rest.blacklist', [ -// 'license/30_enterprise_license/Installing enterprise license', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', -// 'ml/datafeed_cat_apis/Test cat datafeeds', -// 'ml/datafeeds_crud/Test update datafeed to point to different job', -// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', -// 'ml/datafeeds_crud/Test update datafeed to point to missing job', -// 'ml/job_cat_apis/Test cat anomaly detector jobs', -// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', -// 'ml/jobs_get_stats/Test get job stats for closed job', -// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', -// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', -// 'ml/post_data/Test flush with skip_time', -// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', -// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', -// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', -// 'ml/trained_model_cat_apis/Test cat trained models', -// 'privileges/11_builtin/Test get builtin privileges', -// 'roles/11_idx_arrays/Test put role api using as array of index names', -// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', -// 'rollup/delete_job/Test basic delete_job', -// 'rollup/delete_job/Test delete job twice', -// 'rollup/delete_job/Test delete running job', -// 'rollup/get_jobs/Test basic get_jobs', -// 'rollup/put_job/Test basic put_job', -// 'rollup/put_job/Test put_job in non-rollup index', -// 'rollup/rollup_search/Obsolete BWC Timezone', -// 'rollup/rollup_search/Obsolete Timezone', -// 'rollup/start_job/Test start job twice', -// 'security/authz/14_cat_indices/Test empty request while no-authorized index', -// 'security/authz/14_cat_indices/Test empty request while single authorized index', -// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', -// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', -// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', -// 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', -// 'sql/sql/Clean cursor', -// 'sql/sql/Execute some SQL', -// 'sql/sql/Getting textual representation', -// 'sql/sql/Paging through results', -// 'sql/translate/Translate SQL', -// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', -// 'transform/transforms_cat_apis/Test cat transform stats with column selection', -// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', -// 'vectors/10_dense_vector_basic/Cosine Similarity', -// 'vectors/10_dense_vector_basic/Deprecated function signature', -// 'vectors/10_dense_vector_basic/Dot Product', -// 'vectors/15_dense_vector_l1l2/L1 norm', -// 'vectors/15_dense_vector_l1l2/L2 norm', -//// 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', -//// 'vectors/20_dense_vector_special_cases/Documents missing a vector field', -//// 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', -//// 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', -//// 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', -// 'vectors/30_sparse_vector_basic/Cosine Similarity', -// 'vectors/30_sparse_vector_basic/Deprecated function signature', -// 'vectors/30_sparse_vector_basic/Dot Product', -// 'vectors/35_sparse_vector_l1l2/L1 norm', -// 'vectors/35_sparse_vector_l1l2/L2 norm', -// 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', -// 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', -// 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', -// 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', -// 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', -// 'vectors/50_vector_stats/Usage stats on vector fields', -// ].join(',') -//} +tasks.named("yamlRestCompatTest").configure { + + systemProperty 'tests.rest.blacklist', [ + 'license/30_enterprise_license/Installing enterprise license', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'ml/trained_model_cat_apis/Test cat trained models', + 'privileges/11_builtin/Test get builtin privileges', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', + 'rollup/put_job/Test put_job in non-rollup index', + 'rollup/rollup_search/Obsolete BWC Timezone', + 'rollup/rollup_search/Obsolete Timezone', + 'rollup/start_job/Test start job twice', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'sql/sql/Clean cursor', + 'sql/sql/Execute some SQL', + 'sql/sql/Getting textual representation', + 'sql/sql/Paging through results', + 'sql/translate/Translate SQL', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields' + ].join(',') +} tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") From 9e1644f8d4f2db1da95d85843d503ddc64a498b4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 12 Apr 2021 15:19:53 +0200 Subject: [PATCH 22/32] review follow up. remove the type checks in rollover request --- .../rest/compat/RestCompatTestTransformTask.java | 9 --------- .../admin/indices/rollover/RolloverRequest.java | 14 ++------------ .../elasticsearch/index/mapper/MapperService.java | 8 -------- .../indices/rollover/RolloverRequestTests.java | 8 +------- 4 files changed, 3 insertions(+), 36 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 2476721ca1e36..1128417b9dc00 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -24,7 +24,6 @@ import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; -import org.elasticsearch.gradle.test.rest.transform.match.ReplaceTextual; import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsFalse; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsTrue; @@ -106,14 +105,6 @@ public void replaceMatch(String subKey, Object value) { transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } - public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); - } - - public void replaceIsFalse(String subKey, Object value) { - transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); - } - /** * Replaces all the values of a is_true assertion for all project REST tests. * For example "is_true": "value_to_replace" to "match": "value_replaced" diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index b2d49b85411e0..a5b9f4c98f3d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -76,21 +75,12 @@ public class RolloverRequest extends AcknowledgedRequest implem } } else { // a type is not included, add a dummy _doc type - Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + - "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); - } - request.createIndexRequest.mapping(mappings); + request.createIndexRequest.mapping(parser.map()); } }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type - Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); - } - request.createIndexRequest.mapping(mappings); + request.createIndexRequest.mapping(parser.map()); }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 82d7a49531dd0..83937f19468ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -332,14 +332,6 @@ public DocumentMapper documentMapper() { public static boolean isMappingSourceTyped(String type, Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } - /** - * Returns {@code true} if the given {@code mappingSource} includes any type (anything other than properties) - * as a top-level object. - */ - public static boolean isMappingSourceTyped(Map mapping) { - return mapping.size() == 1 && - mapping.keySet().iterator().next().equals("properties") == false; - } /** * Resolves a type from a mapping-related request into the type that should be used when diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 4dc413b5fb834..8b4525d58a155 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -197,7 +197,7 @@ public void testParsingWithType() throws Exception { .field("max_docs", 100) .endObject() .startObject("mappings") - .startObject("type1") //TO ASK did we allow types or only _doc? + .startObject("type1") .startObject("properties") .startObject("field1") .field("type", "string") @@ -223,12 +223,6 @@ public void testParsingWithType() throws Exception { assertThat(request.getCreateIndexRequest().mappings(), equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"index\":\"not_analyzed\",\"type\":\"string\"}}}}")); } - - try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, - BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { - final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); - expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); - } } private static List> conditionsGenerator = new ArrayList<>(); From 2049d0e95cb41a890fb8eef2b3902bb0263d2a4c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 13 Apr 2021 09:27:15 +0200 Subject: [PATCH 23/32] bring back a check against type without include_type_name --- .../indices/rollover/RolloverRequest.java | 8 ++++++- .../rollover/RolloverRequestTests.java | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index a5b9f4c98f3d8..22ff5921917c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -75,7 +76,12 @@ public class RolloverRequest extends AcknowledgedRequest implem } } else { // a type is not included, add a dummy _doc type - request.createIndexRequest.mapping(parser.map()); + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); + } + request.createIndexRequest.mapping(mappings); } }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 8b4525d58a155..7269c68a3d3a8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -225,6 +225,27 @@ public void testParsingWithType() throws Exception { } } + public void testTypedRequestWithoutIncludeTypeName() throws IOException { + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("mappings") + .startObject("_doc") + .startObject("properties") + .startObject("field1") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); + } + } + private static List> conditionsGenerator = new ArrayList<>(); static { conditionsGenerator.add((request) -> request.addMaxIndexDocsCondition(randomNonNegativeLong())); From a1ea94cb8080d6ce00cc95f57d11cd004704427b Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 13 Apr 2021 16:09:53 +0200 Subject: [PATCH 24/32] remove stale gradle transformations --- .../rest/transform/match/ReplaceTextual.java | 85 -------------- .../transform/match/ReplaceTextualTests.java | 107 ----------------- .../rest/transform/match/text_replace.yml | 110 ------------------ 3 files changed, 302 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java delete mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java delete mode 100644 buildSrc/src/test/resources/rest/transform/match/text_replace.yml diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java deleted file mode 100644 index ad32bc3bb86c5..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.Optional; - -/** - * A transformation to replace the flat textual fields. - */ -public class ReplaceTextual implements RestTestTransformByParentObject { - private final String keyToReplaceName; - private final String valueToBeReplaced; - private final TextNode replacementNode; - private final String testName; - - public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode) { - this.keyToReplaceName = keyToReplaceName; - this.valueToBeReplaced = valueToBeReplaced; - this.replacementNode = replacementNode; - this.testName = null; - } - - public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode, String testName) { - this.keyToReplaceName = keyToReplaceName; - this.valueToBeReplaced = valueToBeReplaced; - this.replacementNode = replacementNode; - this.testName = testName; - } - - @Override - @Internal - public String getKeyToFind() { - return keyToReplaceName; - } - - @Override - public String requiredChildKey() { - return valueToBeReplaced; - } - - @Override - public boolean shouldApply(RestTestContext testContext) { - return testName == null || testContext.getTestName().equals(testName); - } - - @Override - public void transformTest(ObjectNode matchParent) { - matchParent.set(getKeyToFind(), replacementNode); - } - - @Input - public String getValueToBeReplaced() { - return valueToBeReplaced; - } - - @Input - public JsonNode getReplacementNode() { - return replacementNode; - } - - @Input - @Optional - public String getTestName() { - return testName; - } - - @Override - public boolean matches(JsonNode child) { - return child.asText().equals(requiredChildKey()); - } - -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java deleted file mode 100644 index 0f05a4ba2cc7d..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; -import org.hamcrest.CoreMatchers; -import org.junit.Test; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public class ReplaceTextualTests extends TransformTests { - - private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); - private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); - - @Test - public void testReplaceAll() throws Exception { - String testName = "/rest/transform/match/text_replace.yml"; - List tests = getTests(testName); - TextNode replacementNode = MAPPER.convertValue("_replaced_value", TextNode.class); - validateTest(tests, true, true); - List transformedTests = transformTests( - new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("key_to_replace", "value_to_replace", replacementNode, null)) - ); - printTest(testName, transformedTests); - validateTest(tests, false, true); - } - - private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { - validateSetupAndTearDownForMatchTests(tests); - // first test - JsonNode firstTestChild = tests.get(2).get("First test"); - assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; - - AtomicBoolean firstTestOccurrenceFound = new AtomicBoolean(false); - - firstTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - firstTestOccurrenceFound.set(true); - if (beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); - } - } - }); - assertTrue(firstTestOccurrenceFound.get()); - - // last test - JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); - assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; - - AtomicBoolean lastTestOccurrenceFound = new AtomicBoolean(false); - lastTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - lastTestOccurrenceFound.set(true); - if (beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); - } - } - }); - assertTrue(lastTestOccurrenceFound.get()); - - // exclude setup, teardown, first test, and last test - for (int i = 3; i <= tests.size() - 2; i++) { - ObjectNode otherTest = tests.get(i); - JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); - assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; - otherTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null && matchObject.get("key_to_replace") != null && beforeTransformation == false && allTests) { - assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); - } - }); - } - } - - @Override - protected boolean getHumanDebug() { - return true; - } -} diff --git a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml deleted file mode 100644 index a3f71bc33ffd0..0000000000000 --- a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -setup: - - do: - something: - here: ok ---- -teardown: - - do: - something_else: - here: true ---- -"First test": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { copied.from.real.test.total: 1 } - - match: { hits.hits.0._index: "single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _shards.failed: 0 } - - - do: - and: again - - - match: { hits.total: 1 } - - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _below_is_target_for_tests: 0 } - - key_to_replace: "value_to_replace" - ---- -"Also has _type match": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - key_to_replace: "value_to_replace" - - match: { _below_is_target_for_tests: 0 } - - match: { _type: foo } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - not_random_but_representive: "of actual test" - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - ---- -"Does not have _type match ": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - it: again - - - match: { _type: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } ---- -"Last test": - - - do: - something: 中文 - - - match: { _index: test_1 } - - key_to_replace: "value_to_replace" - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - - do: - something_else: 中文 - - - match: { _index: test_1 } - - match: { _type: "the value does not matter" } - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - From 5f1b95025d2a17d1d12e313e52377e5fc0ecf441 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 14 Apr 2021 14:09:40 +0200 Subject: [PATCH 25/32] missing version quard check --- .../action/admin/indices/RestGetFieldMappingAction.java | 8 +++----- .../action/admin/indices/RestGetIndexTemplateAction.java | 6 ++---- .../rest/action/admin/indices/RestGetMappingAction.java | 3 ++- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 530198a5b7136..dbad6054d2023 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -58,11 +58,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); - if(request.getRestApiVersion() == RestApiVersion.V_7){ - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); - } + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index b3379899a6f40..4d42c371c591d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -56,10 +56,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if(request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); - } + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); } final String[] names = Strings.splitStringByCommaToArray(request.param("name")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index a6d271991a2bc..96bfb63d45abf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; @@ -59,7 +60,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { request.param(INCLUDE_TYPE_NAME_PARAMETER); deprecationLogger.compatibleApiWarning("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } From 95bba64fe338b320636709d1aa08c48cff5ed25d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 10:49:09 +0200 Subject: [PATCH 26/32] precommit --- .../action/admin/indices/rollover/RolloverRequest.java | 7 ++++++- .../cluster/metadata/IndexTemplateMetadata.java | 7 +++++-- .../main/java/org/elasticsearch/rest/BaseRestHandler.java | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 22ff5921917c9..31a40b22a9e4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -86,7 +86,12 @@ public class RolloverRequest extends AcknowledgedRequest implem }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type - request.createIndexRequest.mapping(parser.map()); + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + + throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); + } + request.createIndexRequest.mapping(mappings); }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java index ef592c48b2a14..fff997e4bc9ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; @@ -38,6 +37,10 @@ import java.util.Objects; import java.util.Set; +import static org.elasticsearch.common.RestApiVersion.V_8; +import static org.elasticsearch.common.RestApiVersion.onOrAfter; + + public class IndexTemplateMetadata extends AbstractDiffable { private final String name; @@ -379,7 +382,7 @@ private static void toInnerXContent(IndexTemplateMetadata indexTemplateMetadata, indexTemplateMetadata.settings().toXContent(builder, params); builder.endObject(); - if(builder.getRestApiVersion() != RestApiVersion.V_7) { + if(builder.getRestApiVersion().matches(onOrAfter(V_8))) { includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); } diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 2d2d0cc83a0d3..857e92bdbc18a 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -43,7 +43,7 @@ public abstract class BaseRestHandler implements RestHandler { /** * Parameter that controls whether certain REST apis should include type names in their requests or responses. - * Note: This parameter is only available through compatible rest api. + * Note: This parameter is only available through compatible rest api for {@link RestApiVersion#V_7}. */ public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; From 2199c3e38942c005ea522d073805d2c9415dbd51 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 15:57:23 +0200 Subject: [PATCH 27/32] code review follow up --- rest-api-spec/build.gradle | 2 -- .../admin/indices/template/get/GetIndexTemplatesResponse.java | 3 +-- .../rest/action/admin/indices/RestGetFieldMappingAction.java | 2 +- .../rest/action/admin/indices/RestRolloverIndexAction.java | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 079359fa5e71f..28ae195ed16cc 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -221,9 +221,7 @@ tasks.named("yamlRestCompatTest").configure { 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', 'indices.put_mapping/all_path_options_with_types/put one mapping per index', - // needs discussion about IndexTemplateMetadata#toInnerXContent and reducingMappings when includeTypeName and empty mappings etc 'indices.put_template/11_basic_with_types/Put template with empty mappings', - // 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', 'indices.shard_stores/10_basic/basic index test', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index f54fc070a3425..40ad35fd1eb5a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -21,7 +21,6 @@ import java.util.Objects; import static java.util.Collections.singletonMap; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject { @@ -69,7 +68,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { if(builder.getRestApiVersion() == RestApiVersion.V_7 && - params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { + params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false)) { IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); } else { IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index dbad6054d2023..3c1ae91d42123 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -59,7 +59,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + request.param(INCLUDE_TYPE_NAME_PARAMETER); deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 0c4cf0c9e2f5a..d8e0c329702f6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -42,7 +42,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final boolean includeTypeName = isIncludeTypeName(request); + final boolean includeTypeName = includeTypeName(request); RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); @@ -53,7 +53,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel)); } - private boolean isIncludeTypeName(RestRequest request) { + private boolean includeTypeName(RestRequest request) { boolean includeTypeName = false; if (request.getRestApiVersion() == RestApiVersion.V_7) { if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { From 8c57e85f392a48dc00723ea91d0fff2e1182e49e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 17:06:18 +0200 Subject: [PATCH 28/32] keep the test ignored --- rest-api-spec/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 28ae195ed16cc..71f546af3f314 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -221,6 +221,7 @@ tasks.named("yamlRestCompatTest").configure { 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', 'indices.put_mapping/all_path_options_with_types/put one mapping per index', + // there is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003 'indices.put_template/11_basic_with_types/Put template with empty mappings', 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', From a571491661b642046348c4362f775ddb77985ca1 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Apr 2021 09:36:26 +0200 Subject: [PATCH 29/32] tweak comment --- rest-api-spec/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 71f546af3f314..8634b54dba8d8 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -222,6 +222,7 @@ tasks.named("yamlRestCompatTest").configure { 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', 'indices.put_mapping/all_path_options_with_types/put one mapping per index', // there is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003 + // not fixing this in #70966 'indices.put_template/11_basic_with_types/Put template with empty mappings', 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', From 75edcd3a334bfdc2c459c62b64dd787819e473bd Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Apr 2021 10:13:29 +0200 Subject: [PATCH 30/32] enable some xpack plugin tests --- x-pack/plugin/build.gradle | 121 +++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 66 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 3ae05b92684ff..0ec2ff4053ebd 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -98,72 +98,61 @@ tasks.named("yamlRestCompatTest").configure { //TODO: blacklist specific to REST API compatibility restTestBlacklist.addAll([ - 'license/30_enterprise_license/Installing enterprise license', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'ml/post_data/Test flush with skip_time', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'ml/trained_model_cat_apis/Test cat trained models', - 'privileges/11_builtin/Test get builtin privileges', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'rollup/delete_job/Test basic delete_job', - 'rollup/delete_job/Test delete job twice', - 'rollup/delete_job/Test delete running job', - 'rollup/get_jobs/Test basic get_jobs', - 'rollup/put_job/Test basic put_job', - 'rollup/put_job/Test put_job in non-rollup index', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'rollup/rollup_search/Obsolete Timezone', - 'rollup/start_job/Test start job twice', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', - 'sql/sql/Clean cursor', - 'sql/sql/Execute some SQL', - 'sql/sql/Getting textual representation', - 'sql/sql/Paging through results', - 'sql/translate/Translate SQL', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'vectors/10_dense_vector_basic/Cosine Similarity', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/10_dense_vector_basic/Dot Product', - 'vectors/15_dense_vector_l1l2/L1 norm', - 'vectors/15_dense_vector_l1l2/L2 norm', - 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', - 'vectors/20_dense_vector_special_cases/Documents missing a vector field', - 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', - 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', - 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/35_sparse_vector_l1l2/L1 norm', - 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/50_vector_stats/Usage stats on vector fields' + 'license/30_enterprise_license/Installing enterprise license', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'ml/trained_model_cat_apis/Test cat trained models', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', + 'rollup/put_job/Test put_job in non-rollup index', + 'rollup/rollup_search/Obsolete BWC Timezone', + 'rollup/rollup_search/Obsolete Timezone', + 'rollup/start_job/Test start job twice', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'sql/sql/Clean cursor', + 'sql/sql/Execute some SQL', + 'sql/sql/Getting textual representation', + 'sql/sql/Paging through results', + 'sql/translate/Translate SQL', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields' ]) systemProperty 'tests.rest.blacklist', restTestBlacklist.join(',') From 5f2620b59fe54687cfb9b72cbebd9544aebe16ec Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Apr 2021 13:23:29 +0200 Subject: [PATCH 31/32] fix xpack-plugin block list --- x-pack/plugin/build.gradle | 64 +------------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 0ec2ff4053ebd..45c4ffd12682b 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -147,7 +147,7 @@ tasks.named("yamlRestCompatTest").configure { 'vectors/30_sparse_vector_basic/Dot Product', 'vectors/35_sparse_vector_l1l2/L1 norm', 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently' 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', @@ -206,68 +206,6 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -tasks.named("yamlRestCompatTest").configure { - - systemProperty 'tests.rest.blacklist', [ - 'license/30_enterprise_license/Installing enterprise license', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/post_data/Test flush with skip_time', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'ml/trained_model_cat_apis/Test cat trained models', - 'privileges/11_builtin/Test get builtin privileges', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'rollup/delete_job/Test basic delete_job', - 'rollup/delete_job/Test delete job twice', - 'rollup/delete_job/Test delete running job', - 'rollup/get_jobs/Test basic get_jobs', - 'rollup/put_job/Test basic put_job', - 'rollup/put_job/Test put_job in non-rollup index', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'rollup/rollup_search/Obsolete Timezone', - 'rollup/start_job/Test start job twice', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'sql/sql/Clean cursor', - 'sql/sql/Execute some SQL', - 'sql/sql/Getting textual representation', - 'sql/sql/Paging through results', - 'sql/translate/Translate SQL', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/35_sparse_vector_l1l2/L1 norm', - 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/50_vector_stats/Usage stats on vector fields' - ].join(',') -} - tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") From 063d0437d859be8237ee654a63c140e9f2c1e359 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Apr 2021 14:15:26 +0200 Subject: [PATCH 32/32] fix build --- x-pack/plugin/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 45c4ffd12682b..6372256ac9ab6 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -147,7 +147,7 @@ tasks.named("yamlRestCompatTest").configure { 'vectors/30_sparse_vector_basic/Dot Product', 'vectors/35_sparse_vector_l1l2/L1 norm', 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently' + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions',