Skip to content

Commit

Permalink
Adding deprecation checks for geo_shape parameters (#76627)
Browse files Browse the repository at this point in the history
The following properties have been removed from the geo_shape data type in 8.0: "strategy",
"tree", "tree_levels", "precision", "distance_error_pct", "points_only". This PR adds a deprecation
info API check for indexes and templates that have any of these properties in their mappings.

Relates #42404 #70850
  • Loading branch information
masseyke authored Sep 1, 2021
1 parent 0877bbc commit 121bd05
Show file tree
Hide file tree
Showing 5 changed files with 279 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,32 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.ingest.PipelineConfiguration;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING;
import static org.elasticsearch.search.SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING;
Expand Down Expand Up @@ -201,4 +208,86 @@ static DeprecationIssue checkClusterRoutingAllocationIncludeRelocationsSetting(f
DeprecationIssue.Level.WARNING
);
}

@SuppressWarnings("unchecked")
private static String getDetailsMessageForComponentTemplates(Map<String, ComponentTemplate> componentTemplates) {
String detailsForComponentTemplates =
componentTemplates.entrySet().stream().map((templateCursor) -> {
String templateName = templateCursor.getKey();
ComponentTemplate componentTemplate = templateCursor.getValue();
CompressedXContent mappings = componentTemplate.template().mappings();
if (mappings != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mappings.uncompressed(), true,
XContentType.JSON);
Map<String, Object> mappingAsMap = tuple.v2();
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
mappingAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
if (messages.isEmpty() == false) {
String messageForMapping =
"mappings in component template " + templateName + " contains deprecated geo_shape properties. " +
messages.stream().collect(Collectors.joining("; "));
return messageForMapping;
}
}
return null;
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
return detailsForComponentTemplates;
}

@SuppressWarnings("unchecked")
private static String getDetailsMessageForIndexTemplates(ImmutableOpenMap<String, IndexTemplateMetadata> indexTemplates) {
String detailsForIndexTemplates =
StreamSupport.stream(indexTemplates.spliterator(), false).map((templateCursor) -> {
String templateName = templateCursor.key;
IndexTemplateMetadata indexTemplateMetadata = templateCursor.value;
String messageForTemplate =
StreamSupport.stream(indexTemplateMetadata.getMappings().spliterator(), false).map((mappingCursor) -> {
CompressedXContent mapping = mappingCursor.value;
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mapping.uncompressed(), true,
XContentType.JSON);
Map<String, Object> mappingAsMap = (Map<String, Object>) tuple.v2().get("_doc");
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
mappingAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
return messages;
}).filter(messages -> messages.isEmpty() == false).map(messages -> {
String messageForMapping =
"mappings in index template " + templateName + " contains deprecated geo_shape properties. " +
messages.stream().collect(Collectors.joining("; "));
return messageForMapping;
}).collect(Collectors.joining("; "));
return messageForTemplate;
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
return detailsForIndexTemplates;
}

@SuppressWarnings("unchecked")
static DeprecationIssue checkGeoShapeTemplates(final ClusterState clusterState) {
String detailsForComponentTemplates = getDetailsMessageForComponentTemplates(clusterState.getMetadata().componentTemplates());
String detailsForIndexTemplates = getDetailsMessageForIndexTemplates(clusterState.getMetadata().getTemplates());
boolean deprecationInComponentTemplates = Strings.isEmpty(detailsForComponentTemplates) == false;
boolean deprecationInIndexTemplates = Strings.isEmpty(detailsForIndexTemplates) == false;
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
if (deprecationInComponentTemplates && deprecationInIndexTemplates) {
String message = "component templates and index templates contain deprecated geo_shape properties that must be removed";
String details = detailsForComponentTemplates + "; " + detailsForIndexTemplates;
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false,
null);
} if (deprecationInComponentTemplates == false && deprecationInIndexTemplates) {
String message = "index templates contain deprecated geo_shape properties that must be removed";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForIndexTemplates, false,
null);
} else if (deprecationInIndexTemplates == false && deprecationInComponentTemplates) {
String message = "component templates contain deprecated geo_shape properties that must be removed";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForComponentTemplates, false,
null);
} else {
return null;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ private DeprecationChecks() {
ClusterDeprecationChecks::checkPollIntervalTooLow,
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled,
ClusterDeprecationChecks::checkTemplatesWithMultipleTypes,
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting,
ClusterDeprecationChecks::checkGeoShapeTemplates
));

static final List<NodeDeprecationCheck<Settings, PluginsAndModules, ClusterState, XPackLicenseState, DeprecationIssue>>
Expand Down Expand Up @@ -111,7 +112,8 @@ private DeprecationChecks() {
IndexDeprecationChecks::checkIndexDataPath,
IndexDeprecationChecks::indexingSlowLogLevelSettingCheck,
IndexDeprecationChecks::searchSlowLogLevelSettingCheck,
IndexDeprecationChecks::storeTypeSettingCheck
IndexDeprecationChecks::storeTypeSettingCheck,
IndexDeprecationChecks::checkGeoShapeMappings
));

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.SlowLogLevel;
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;

import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -31,6 +32,7 @@
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;


/**
Expand Down Expand Up @@ -317,4 +319,42 @@ static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) {
}
return null;
}

protected static boolean isGeoShapeFieldWithDeprecatedParam(Map<?, ?> property) {
return LegacyGeoShapeFieldMapper.CONTENT_TYPE.equals(property.get("type")) &&
LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream().anyMatch(deprecatedParameter ->
property.containsKey(deprecatedParameter)
);
}

protected static String formatDeprecatedGeoShapeParamMessage(String type, Map.Entry<?, ?> entry) {
String fieldName = entry.getKey().toString();
Map<?, ?> value = (Map<?, ?>) entry.getValue();
return LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream()
.filter(deprecatedParameter -> value.containsKey(deprecatedParameter))
.map(deprecatedParameter -> String.format(Locale.ROOT, "parameter [%s] in field [%s]", type, deprecatedParameter, fieldName))
.collect(Collectors.joining("; "));
}

@SuppressWarnings("unchecked")
static DeprecationIssue checkGeoShapeMappings(IndexMetadata indexMetadata) {
if (indexMetadata == null || indexMetadata.mapping() == null) {
return null;
}
Map<String, Object> sourceAsMap = indexMetadata.mapping().getSourceAsMap();
List<String> messages = findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, sourceAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
if (messages.isEmpty()) {
return null;
} else {
String message = String.format(Locale.ROOT,"mappings for index %s contains deprecated geo_shape properties that must be " +
"removed", indexMetadata.getIndex().getName());
String details = String.format(Locale.ROOT,
"The following geo_shape parameters must be removed from %s: [%s]", indexMetadata.getIndex().getName(),
messages.stream().collect(Collectors.joining("; ")));
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, null);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
Expand All @@ -24,6 +28,7 @@

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;

Expand Down Expand Up @@ -355,4 +360,83 @@ public void testClusterRoutingAllocationIncludeRelocationsSetting() {

assertWarnings(expectedWarning);
}

public void testCheckGeoShapeMappings() throws Exception {
// First, testing only an index template:
IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("single-type")
.patterns(Collections.singletonList("foo"))
.putMapping("_doc", "{\n" +
" \"_doc\":{\n" +
" \"properties\":{\n" +
" \"nested_field\":{\n" +
" \"type\":\"nested\",\n" +
" \"properties\":{\n" +
" \"location\":{\n" +
" \"type\":\"geo_shape\",\n" +
" \"strategy\":\"recursive\",\n" +
" \"points_only\":true\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}")
.build();
ImmutableOpenMap<String, IndexTemplateMetadata> templates = ImmutableOpenMap.<String, IndexTemplateMetadata>builder()
.fPut("single-type", indexTemplateMetadata)
.build();
Metadata badMetadata = Metadata.builder()
.templates(templates)
.build();
ClusterState badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
DeprecationIssue issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"index templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in index template single-type contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
));

// Second, testing only a component template:
String templateName = "my-template";
Settings settings = Settings.builder().put("index.number_of_shards", 1).build();
CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"location\":{\"type\":\"geo_shape\", " +
"\"strategy\":\"recursive\", \"points_only\":true}}}");
AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build();
Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias));
ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
badMetadata = Metadata.builder()
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
.build();
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"component templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
));

// Third, trying a component template and an index template:
badMetadata = Metadata.builder()
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
.templates(templates)
.build();
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"component templates and index templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]; mappings in index template single-type contains " +
"deprecated geo_shape properties. [parameter [geo_shape] in field [points_only]; parameter [geo_shape] in field " +
"[strategy]]", false, null)
));
}
}
Loading

0 comments on commit 121bd05

Please sign in to comment.