From 121bd0580a9dc2f4e7771838925e09962cf70cfc Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 1 Sep 2021 16:00:21 -0500 Subject: [PATCH] Adding deprecation checks for geo_shape parameters (#76627) The following properties have been removed from the geo_shape data type in 8.0: "strategy", "tree", "tree_levels", "precision", "distance_error_pct", "points_only". This PR adds a deprecation info API check for indexes and templates that have any of these properties in their mappings. Relates #42404 #70850 --- .../deprecation/ClusterDeprecationChecks.java | 89 +++++++++++++++++++ .../xpack/deprecation/DeprecationChecks.java | 6 +- .../deprecation/IndexDeprecationChecks.java | 40 +++++++++ .../ClusterDeprecationChecksTests.java | 84 +++++++++++++++++ .../IndexDeprecationChecksTests.java | 62 +++++++++++++ 5 files changed, 279 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index 84910d26ef5ce..050bdd0066032 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -10,18 +10,24 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.PipelineConfiguration; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -29,6 +35,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING; import static org.elasticsearch.search.SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING; @@ -201,4 +208,86 @@ static DeprecationIssue checkClusterRoutingAllocationIncludeRelocationsSetting(f DeprecationIssue.Level.WARNING ); } + + @SuppressWarnings("unchecked") + private static String getDetailsMessageForComponentTemplates(Map componentTemplates) { + String detailsForComponentTemplates = + componentTemplates.entrySet().stream().map((templateCursor) -> { + String templateName = templateCursor.getKey(); + ComponentTemplate componentTemplate = templateCursor.getValue(); + CompressedXContent mappings = componentTemplate.template().mappings(); + if (mappings != null) { + Tuple> tuple = XContentHelper.convertToMap(mappings.uncompressed(), true, + XContentType.JSON); + Map mappingAsMap = tuple.v2(); + List messages = mappingAsMap == null ? Collections.emptyList() : + IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, + mappingAsMap, + IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam, + IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage); + if (messages.isEmpty() == false) { + String messageForMapping = + "mappings in component template " + templateName + " contains deprecated geo_shape properties. " + + messages.stream().collect(Collectors.joining("; ")); + return messageForMapping; + } + } + return null; + }).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; ")); + return detailsForComponentTemplates; + } + + @SuppressWarnings("unchecked") + private static String getDetailsMessageForIndexTemplates(ImmutableOpenMap indexTemplates) { + String detailsForIndexTemplates = + StreamSupport.stream(indexTemplates.spliterator(), false).map((templateCursor) -> { + String templateName = templateCursor.key; + IndexTemplateMetadata indexTemplateMetadata = templateCursor.value; + String messageForTemplate = + StreamSupport.stream(indexTemplateMetadata.getMappings().spliterator(), false).map((mappingCursor) -> { + CompressedXContent mapping = mappingCursor.value; + Tuple> tuple = XContentHelper.convertToMap(mapping.uncompressed(), true, + XContentType.JSON); + Map mappingAsMap = (Map) tuple.v2().get("_doc"); + List messages = mappingAsMap == null ? Collections.emptyList() : + IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, + mappingAsMap, + IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam, + IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage); + return messages; + }).filter(messages -> messages.isEmpty() == false).map(messages -> { + String messageForMapping = + "mappings in index template " + templateName + " contains deprecated geo_shape properties. " + + messages.stream().collect(Collectors.joining("; ")); + return messageForMapping; + }).collect(Collectors.joining("; ")); + return messageForTemplate; + }).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; ")); + return detailsForIndexTemplates; + } + + @SuppressWarnings("unchecked") + static DeprecationIssue checkGeoShapeTemplates(final ClusterState clusterState) { + String detailsForComponentTemplates = getDetailsMessageForComponentTemplates(clusterState.getMetadata().componentTemplates()); + String detailsForIndexTemplates = getDetailsMessageForIndexTemplates(clusterState.getMetadata().getTemplates()); + boolean deprecationInComponentTemplates = Strings.isEmpty(detailsForComponentTemplates) == false; + boolean deprecationInIndexTemplates = Strings.isEmpty(detailsForIndexTemplates) == false; + String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes"; + if (deprecationInComponentTemplates && deprecationInIndexTemplates) { + String message = "component templates and index templates contain deprecated geo_shape properties that must be removed"; + String details = detailsForComponentTemplates + "; " + detailsForIndexTemplates; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, + null); + } if (deprecationInComponentTemplates == false && deprecationInIndexTemplates) { + String message = "index templates contain deprecated geo_shape properties that must be removed"; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForIndexTemplates, false, + null); + } else if (deprecationInIndexTemplates == false && deprecationInComponentTemplates) { + String message = "component templates contain deprecated geo_shape properties that must be removed"; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForComponentTemplates, false, + null); + } else { + return null; + } + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index dddccceb358ac..06eb504b7a61c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -38,7 +38,8 @@ private DeprecationChecks() { ClusterDeprecationChecks::checkPollIntervalTooLow, ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled, ClusterDeprecationChecks::checkTemplatesWithMultipleTypes, - ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting + ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting, + ClusterDeprecationChecks::checkGeoShapeTemplates )); static final List> @@ -111,7 +112,8 @@ private DeprecationChecks() { IndexDeprecationChecks::checkIndexDataPath, IndexDeprecationChecks::indexingSlowLogLevelSettingCheck, IndexDeprecationChecks::searchSlowLogLevelSettingCheck, - IndexDeprecationChecks::storeTypeSettingCheck + IndexDeprecationChecks::storeTypeSettingCheck, + IndexDeprecationChecks::checkGeoShapeMappings )); /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index 37ffde62a06f6..8ccc63a51964c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexingSlowLog; import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.SlowLogLevel; +import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import java.util.ArrayList; import java.util.Collections; @@ -31,6 +32,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; /** @@ -317,4 +319,42 @@ static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) { } return null; } + + protected static boolean isGeoShapeFieldWithDeprecatedParam(Map property) { + return LegacyGeoShapeFieldMapper.CONTENT_TYPE.equals(property.get("type")) && + LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream().anyMatch(deprecatedParameter -> + property.containsKey(deprecatedParameter) + ); + } + + protected static String formatDeprecatedGeoShapeParamMessage(String type, Map.Entry entry) { + String fieldName = entry.getKey().toString(); + Map value = (Map) entry.getValue(); + return LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream() + .filter(deprecatedParameter -> value.containsKey(deprecatedParameter)) + .map(deprecatedParameter -> String.format(Locale.ROOT, "parameter [%s] in field [%s]", type, deprecatedParameter, fieldName)) + .collect(Collectors.joining("; ")); + } + + @SuppressWarnings("unchecked") + static DeprecationIssue checkGeoShapeMappings(IndexMetadata indexMetadata) { + if (indexMetadata == null || indexMetadata.mapping() == null) { + return null; + } + Map sourceAsMap = indexMetadata.mapping().getSourceAsMap(); + List messages = findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, sourceAsMap, + IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam, + IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage); + if (messages.isEmpty()) { + return null; + } else { + String message = String.format(Locale.ROOT,"mappings for index %s contains deprecated geo_shape properties that must be " + + "removed", indexMetadata.getIndex().getName()); + String details = String.format(Locale.ROOT, + "The following geo_shape parameters must be removed from %s: [%s]", indexMetadata.getIndex().getName(), + messages.stream().collect(Collectors.joining("; "))); + String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes"; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, null); + } + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java index 8063b4f9853a3..bbb6c384c2793 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -9,11 +9,15 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -24,6 +28,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -355,4 +360,83 @@ public void testClusterRoutingAllocationIncludeRelocationsSetting() { assertWarnings(expectedWarning); } + + public void testCheckGeoShapeMappings() throws Exception { + // First, testing only an index template: + IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("single-type") + .patterns(Collections.singletonList("foo")) + .putMapping("_doc", "{\n" + + " \"_doc\":{\n" + + " \"properties\":{\n" + + " \"nested_field\":{\n" + + " \"type\":\"nested\",\n" + + " \"properties\":{\n" + + " \"location\":{\n" + + " \"type\":\"geo_shape\",\n" + + " \"strategy\":\"recursive\",\n" + + " \"points_only\":true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}") + .build(); + ImmutableOpenMap templates = ImmutableOpenMap.builder() + .fPut("single-type", indexTemplateMetadata) + .build(); + Metadata badMetadata = Metadata.builder() + .templates(templates) + .build(); + ClusterState badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + DeprecationIssue issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "index templates contain deprecated geo_shape properties that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes", + "mappings in index template single-type contains deprecated geo_shape properties. [parameter [geo_shape] in field " + + "[points_only]; parameter [geo_shape] in field [strategy]]", false, null) + )); + + // Second, testing only a component template: + String templateName = "my-template"; + Settings settings = Settings.builder().put("index.number_of_shards", 1).build(); + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"location\":{\"type\":\"geo_shape\", " + + "\"strategy\":\"recursive\", \"points_only\":true}}}"); + AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build(); + Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias)); + ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); + badMetadata = Metadata.builder() + .componentTemplates(Collections.singletonMap(templateName, componentTemplate)) + .build(); + badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "component templates contain deprecated geo_shape properties that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes", + "mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " + + "[points_only]; parameter [geo_shape] in field [strategy]]", false, null) + )); + + // Third, trying a component template and an index template: + badMetadata = Metadata.builder() + .componentTemplates(Collections.singletonMap(templateName, componentTemplate)) + .templates(templates) + .build(); + badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "component templates and index templates contain deprecated geo_shape properties that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes", + "mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " + + "[points_only]; parameter [geo_shape] in field [strategy]]; mappings in index template single-type contains " + + "deprecated geo_shape properties. [parameter [geo_shape] in field [points_only]; parameter [geo_shape] in field " + + "[strategy]]", false, null) + )); + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java index 205a4a6142da5..f06a251ac56f2 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.joda.JodaDeprecationPatterns; @@ -26,8 +27,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.singletonList; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -497,4 +502,61 @@ public void testSimpleFSSetting() { "as it offers superior or equivalent performance to [simplefs].", false, null) )); } + + public void testCheckGeoShapeMappings() throws Exception { + Map emptyMappingMap = Collections.emptyMap(); + MappingMetadata mappingMetadata = new MappingMetadata("", emptyMappingMap); + Settings.Builder settings = settings(Version.CURRENT); + IndexMetadata indexMetadata = + IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata)); + assertTrue(issues.isEmpty()); + + Map okGeoMappingMap = Collections.singletonMap("properties", Collections.singletonMap("location", + Collections.singletonMap("type", "geo_shape"))); + mappingMetadata = new MappingMetadata("", okGeoMappingMap); + IndexMetadata indexMetadata2 = + IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata2)); + assertTrue(issues.isEmpty()); + + Map deprecatedPropertiesMap = Stream.of(new String[][] { + { "type", "geo_shape" }, + { "strategy", "recursive" }, + { "points_only", "true" } + }).collect(Collectors.toMap(data -> data[0], data -> data[1])); + Map deprecatedGeoMappingMap = Collections.singletonMap("properties", Collections.singletonMap("location", + deprecatedPropertiesMap)); + mappingMetadata = new MappingMetadata("", deprecatedGeoMappingMap); + IndexMetadata indexMetadata3 = + IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata3)); + assertEquals(1, issues.size()); + assertThat(issues, contains( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "mappings for index test contains deprecated geo_shape properties that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes", + "The following geo_shape parameters must be removed from test: [[parameter [geo_shape] in field [points_only]; parameter " + + "[geo_shape] in field [strategy]]]", false, null) + )); + + Map nestedProperties = Stream.of(new Object[][] { + { "type", "nested" }, + { "properties", Collections.singletonMap("location", deprecatedPropertiesMap) }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + Map nestedDeprecatedGeoMappingMap = Collections.singletonMap("properties", + Collections.singletonMap("nested_field", nestedProperties)); + mappingMetadata = new MappingMetadata("", nestedDeprecatedGeoMappingMap); + IndexMetadata indexMetadata4 = + IndexMetadata.builder("test").settings(settings).putMapping(mappingMetadata).numberOfShards(1).numberOfReplicas(0).build(); + issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetadata4)); + assertEquals(1, issues.size()); + assertThat(issues, contains( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "mappings for index test contains deprecated geo_shape properties that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes", + "The following geo_shape parameters must be removed from test: [[parameter [geo_shape] in field [points_only]; parameter " + + "[geo_shape] in field [strategy]]]", false, null) + )); + } }