From e5def5ac82bfe2abc3a2fb418e322f475997e4ad Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 7 Sep 2021 12:27:53 -0500 Subject: [PATCH 1/3] Adding deprecation info API message for sparse_vector --- .../deprecation/ClusterDeprecationChecks.java | 95 ++++++++++++++++++- .../xpack/deprecation/DeprecationChecks.java | 3 +- .../ClusterDeprecationChecksTests.java | 77 +++++++++++++++ 3 files changed, 173 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index 050bdd006603..853966c047b4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -268,7 +269,8 @@ private static String getDetailsMessageForIndexTemplates(ImmutableOpenMap property) { + return "sparse_vector".equals(property.get("type")); + } + + protected static String formatDeprecatedSparseVectorMessage(String type, Map.Entry entry) { + return entry.getKey().toString(); + } + + @SuppressWarnings("unchecked") + private static String getDetailsMessageForSparseVectorComponentTemplates(Map componentTemplates) { + String detailsForComponentTemplates = + componentTemplates.entrySet().stream().map((templateCursor) -> { + String templateName = templateCursor.getKey(); + ComponentTemplate componentTemplate = templateCursor.getValue(); + CompressedXContent mappings = componentTemplate.template().mappings(); + if (mappings != null) { + Tuple> tuple = XContentHelper.convertToMap(mappings.uncompressed(), true, + XContentType.JSON); + Map mappingAsMap = tuple.v2(); + List messages = mappingAsMap == null ? Collections.emptyList() : + IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, + mappingAsMap, + ClusterDeprecationChecks::isSparseVector, + ClusterDeprecationChecks::formatDeprecatedSparseVectorMessage); + if (messages.isEmpty() == false) { + String messageForMapping = + "mappings in component template " + templateName + " contains deprecated sparse_vector fields: " + + messages.stream().collect(Collectors.joining(", ")); + return messageForMapping; + } + } + return null; + }).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; ")); + return detailsForComponentTemplates; + } + + @SuppressWarnings("unchecked") + private static String getDetailsMessageForSparseVectorIndexTemplates(ImmutableOpenMap indexTemplates) { + String detailsForIndexTemplates = + StreamSupport.stream(indexTemplates.spliterator(), false).map((templateCursor) -> { + String templateName = templateCursor.key; + IndexTemplateMetadata indexTemplateMetadata = templateCursor.value; + String messageForTemplate = + StreamSupport.stream(indexTemplateMetadata.getMappings().spliterator(), false).map((mappingCursor) -> { + CompressedXContent mapping = mappingCursor.value; + Tuple> tuple = XContentHelper.convertToMap(mapping.uncompressed(), true, + XContentType.JSON); + Map mappingAsMap = (Map) tuple.v2().get("_doc"); + List messages = mappingAsMap == null ? Collections.emptyList() : + IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, + mappingAsMap, + ClusterDeprecationChecks::isSparseVector, + ClusterDeprecationChecks::formatDeprecatedSparseVectorMessage); + return messages; + }).filter(messages -> messages.isEmpty() == false).map(messages -> { + String messageForMapping = + "mappings in index template " + templateName + " contains deprecated sparse_vector fields: " + + messages.stream().collect(Collectors.joining(", ")); + return messageForMapping; + }).collect(Collectors.joining("; ")); + return messageForTemplate; + }).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; ")); + return detailsForIndexTemplates; + } + + @SuppressWarnings("unchecked") + static DeprecationIssue checkSparseVectorTemplates(final ClusterState clusterState) { + String detailsForComponentTemplates = + getDetailsMessageForSparseVectorComponentTemplates(clusterState.getMetadata().componentTemplates()); + String detailsForIndexTemplates = getDetailsMessageForSparseVectorIndexTemplates(clusterState.getMetadata().getTemplates()); + boolean deprecationInComponentTemplates = Strings.isEmpty(detailsForComponentTemplates) == false; + boolean deprecationInIndexTemplates = Strings.isEmpty(detailsForIndexTemplates) == false; + String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_search_changes"; + if (deprecationInComponentTemplates && deprecationInIndexTemplates) { + String message = "component templates and index templates contain deprecated sparse_vector fields that must be removed"; + String details = detailsForComponentTemplates + "; " + detailsForIndexTemplates; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, + null); + } if (deprecationInComponentTemplates == false && deprecationInIndexTemplates) { + String message = "index templates contain deprecated sparse_vector fields that must be removed"; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForIndexTemplates, false, + null); + } else if (deprecationInIndexTemplates == false && deprecationInComponentTemplates) { + String message = "component templates contain deprecated sparse_vector fields that must be removed"; + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForComponentTemplates, false, + null); + } else { + return null; + } + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index d9da43df6bd9..76f177d4ecdf 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -40,7 +40,8 @@ private DeprecationChecks() { ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled, ClusterDeprecationChecks::checkTemplatesWithMultipleTypes, ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting, - ClusterDeprecationChecks::checkGeoShapeTemplates + ClusterDeprecationChecks::checkGeoShapeTemplates, + ClusterDeprecationChecks::checkSparseVectorTemplates )); static final List> diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java index bbb6c384c279..f822ed704024 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -439,4 +439,81 @@ public void testCheckGeoShapeMappings() throws Exception { "[strategy]]", false, null) )); } + + public void testSparseVectorMappings() throws Exception { + // First, testing only an index template: + IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("single-type") + .patterns(Collections.singletonList("foo")) + .putMapping("_doc", "{\n" + + " \"_doc\":{\n" + + " \"properties\":{\n" + + " \"my_sparse_vector\":{\n" + + " \"type\":\"sparse_vector\"\n" + + " },\n" + + " \"nested_field\":{\n" + + " \"type\":\"nested\",\n" + + " \"properties\":{\n" + + " \"my_nested_sparse_vector\":{\n" + + " \"type\":\"sparse_vector\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}") + .build(); + ImmutableOpenMap templates = ImmutableOpenMap.builder() + .fPut("single-type", indexTemplateMetadata) + .build(); + Metadata badMetadata = Metadata.builder() + .templates(templates) + .build(); + ClusterState badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + DeprecationIssue issue = ClusterDeprecationChecks.checkSparseVectorTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "index templates contain deprecated sparse_vector fields that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_search_changes", + "mappings in index template single-type contains deprecated sparse_vector fields: [my_sparse_vector], " + + "[my_nested_sparse_vector]", false, null) + )); + + // Second, testing only a component template: + String templateName = "my-template"; + Settings settings = Settings.builder().put("index.number_of_shards", 1).build(); + CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"my_sparse_vector\":{\"type\":\"sparse_vector\"}}}"); + AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build(); + Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias)); + ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); + badMetadata = Metadata.builder() + .componentTemplates(Collections.singletonMap(templateName, componentTemplate)) + .build(); + badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + issue = ClusterDeprecationChecks.checkSparseVectorTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "component templates contain deprecated sparse_vector fields that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_search_changes", + "mappings in component template my-template contains deprecated sparse_vector fields: [my_sparse_vector]", false, null) + )); + + // Third, trying a component template and an index template: + badMetadata = Metadata.builder() + .componentTemplates(Collections.singletonMap(templateName, componentTemplate)) + .templates(templates) + .build(); + badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build(); + issue = ClusterDeprecationChecks.checkSparseVectorTemplates(badState); + + assertThat(issue, equalTo( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "component templates and index templates contain deprecated sparse_vector fields that must be removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_search_changes", + "mappings in component template my-template contains deprecated sparse_vector fields: [my_sparse_vector]; " + + "mappings in index template single-type contains deprecated sparse_vector fields: " + + "[my_sparse_vector], [my_nested_sparse_vector]", false, null) + )); + } } From 430250c6286c2d06d571908db26d04039e83c1c3 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 7 Sep 2021 12:38:03 -0500 Subject: [PATCH 2/3] removing unused import --- .../xpack/deprecation/ClusterDeprecationChecks.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index 853966c047b4..e5923bf8897c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; From b861fb02f236efc455237363dd4e50ee1740e9b0 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 7 Sep 2021 15:48:46 -0500 Subject: [PATCH 3/3] Adding punctuation around template name --- .../xpack/deprecation/ClusterDeprecationChecks.java | 2 +- .../xpack/deprecation/ClusterDeprecationChecksTests.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index e5923bf8897c..7d4f12470034 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -318,7 +318,7 @@ private static String getDetailsMessageForSparseVectorComponentTemplates(Map