From d6c6bb222a60d9aabfd466005dc7640faee7bd7e Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 12 Dec 2018 17:40:38 -0700 Subject: [PATCH 1/2] Deprecation check for classic similarity Add a deprecation check for indices which have fields that use the `classic` similarity value. --- .../xpack/deprecation/DeprecationChecks.java | 3 +- .../deprecation/IndexDeprecationChecks.java | 15 ++++++++++ .../IndexDeprecationChecksTests.java | 28 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index ecb4c789ba81f..a73d6a86b3510 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -52,7 +52,8 @@ private DeprecationChecks() { IndexDeprecationChecks::oldIndicesCheck, IndexDeprecationChecks::delimitedPayloadFilterCheck, IndexDeprecationChecks::percolatorUnmappedFieldsAsStringCheck, - IndexDeprecationChecks::indexNameCheck + IndexDeprecationChecks::indexNameCheck, + IndexDeprecationChecks::classicSimilarityCheck )); /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index d95c2508106d5..e9bad471c83be 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -129,4 +129,19 @@ static DeprecationIssue percolatorUnmappedFieldsAsStringCheck(IndexMetaData inde } return null; } + + static DeprecationIssue classicSimilarityCheck(IndexMetaData indexMetaData) { + List issues = new ArrayList<>(); + fieldLevelMappingIssue(indexMetaData, ((mappingMetaData, sourceAsMap) -> issues.addAll( + findInPropertiesRecursively(mappingMetaData.type(), sourceAsMap, + property -> "classic".equals(property.get("similarity")))))); + if (issues.size() > 0) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Classic similarity has been removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + + "#_the_literal_classic_literal_similarity_has_been_removed", + "Fields which use classic similarity: " + issues.toString()); + } + return null; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java index 5b13edbc45e29..f5ec045c6d901 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import java.io.IOException; import java.util.List; import java.util.Locale; @@ -106,4 +107,31 @@ public void testPercolatorUnmappedFieldsAsStringCheck() { List noIssues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(goodIndex)); assertTrue(noIssues.isEmpty()); } + + public void testClassicSimilarityCheck() throws IOException { + String mappingJson = "{\n" + + " \"properties\": {\n" + + " \"default_field\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"classic_sim_field\": {\n" + + " \"type\": \"text\",\n" + + " \"similarity\": \"classic\"\n" + + " }\n" + + " }\n" + + "}"; + IndexMetaData index = IndexMetaData.builder(randomAlphaOfLengthBetween(5,10)) + .settings(settings(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.CURRENT)))) + .numberOfShards(randomIntBetween(1,100)) + .numberOfReplicas(randomIntBetween(1, 100)) + .putMapping("_doc", mappingJson) + .build(); + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Classic similarity has been removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + + "#_the_literal_classic_literal_similarity_has_been_removed", + "Fields which use classic similarity: [[type: _doc, field: classic_sim_field]]"); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(index)); + assertEquals(singletonList(expected), issues); + } } From ea06bec1af7208d9b54a78e4184d0efd0ab61938 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 14 Dec 2018 11:58:47 -0700 Subject: [PATCH 2/2] Add check for classic similarity in settings --- .../xpack/deprecation/DeprecationChecks.java | 3 ++- .../deprecation/IndexDeprecationChecks.java | 19 ++++++++++++++- .../IndexDeprecationChecksTests.java | 24 +++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 8cc4fdff6b381..309fb2e187a0a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -55,7 +55,8 @@ private DeprecationChecks() { IndexDeprecationChecks::indexNameCheck, IndexDeprecationChecks::nodeLeftDelayedTimeCheck, IndexDeprecationChecks::shardOnStartupCheck, - IndexDeprecationChecks::classicSimilarityCheck + IndexDeprecationChecks::classicSimilarityMappingCheck, + IndexDeprecationChecks::classicSimilaritySettingsCheck )); /** diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index f0fd45ad8b13e..b6397db8b9055 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Collectors; /** * Index-specific deprecation checks @@ -134,7 +135,7 @@ static DeprecationIssue percolatorUnmappedFieldsAsStringCheck(IndexMetaData inde return null; } - static DeprecationIssue classicSimilarityCheck(IndexMetaData indexMetaData) { + static DeprecationIssue classicSimilarityMappingCheck(IndexMetaData indexMetaData) { List issues = new ArrayList<>(); fieldLevelMappingIssue(indexMetaData, ((mappingMetaData, sourceAsMap) -> issues.addAll( findInPropertiesRecursively(mappingMetaData.type(), sourceAsMap, @@ -149,6 +150,22 @@ static DeprecationIssue classicSimilarityCheck(IndexMetaData indexMetaData) { return null; } + static DeprecationIssue classicSimilaritySettingsCheck(IndexMetaData indexMetaData) { + Map similarities = indexMetaData.getSettings().getGroups("index.similarity"); + List classicSimilarities = similarities.entrySet().stream() + .filter(entry -> "classic".equals(entry.getValue().get("type"))) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if (classicSimilarities.size() > 0) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Classic similarity has been removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + + "#_the_literal_classic_literal_similarity_has_been_removed", + "Custom similarities defined using classic similarity: " + classicSimilarities.toString()); + } + return null; + } + static DeprecationIssue nodeLeftDelayedTimeCheck(IndexMetaData indexMetaData) { String setting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(); String value = indexMetaData.getSettings().get(setting); diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java index a239d157dc82d..82c1954816fe9 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java @@ -110,7 +110,7 @@ public void testPercolatorUnmappedFieldsAsStringCheck() { assertTrue(noIssues.isEmpty()); } - public void testClassicSimilarityCheck() throws IOException { + public void testClassicSimilarityMappingCheck() throws IOException { String mappingJson = "{\n" + " \"properties\": {\n" + " \"default_field\": {\n" + @@ -123,7 +123,8 @@ public void testClassicSimilarityCheck() throws IOException { " }\n" + "}"; IndexMetaData index = IndexMetaData.builder(randomAlphaOfLengthBetween(5,10)) - .settings(settings(VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.CURRENT)))) + .settings(settings( + VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.CURRENT)))) .numberOfShards(randomIntBetween(1,100)) .numberOfReplicas(randomIntBetween(1, 100)) .putMapping("_doc", mappingJson) @@ -137,6 +138,25 @@ public void testClassicSimilarityCheck() throws IOException { assertEquals(singletonList(expected), issues); } + public void testClassicSimilaritySettingsCheck() { + IndexMetaData index = IndexMetaData.builder(randomAlphaOfLengthBetween(5, 10)) + .settings(settings( + VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.CURRENT))) + .put("index.similarity.my_classic_similarity.type", "classic") + .put("index.similarity.my_okay_similarity.type", "BM25")) + .numberOfShards(randomIntBetween(1, 100)) + .numberOfReplicas(randomIntBetween(1, 100)) + .build(); + + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Classic similarity has been removed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/" + + "#_the_literal_classic_literal_similarity_has_been_removed", + "Custom similarities defined using classic similarity: [my_classic_similarity]"); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(index)); + assertEquals(singletonList(expected), issues); + } + public void testNodeLeftDelayedTimeCheck() { String negativeTimeValue = "-" + randomPositiveTimeValue(); String indexName = randomAlphaOfLengthBetween(0, 10);