From 178b25fc4b3dfc2906e6f09ea031f94ca9517f80 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 21 Sep 2020 10:07:11 +0100 Subject: [PATCH] Fix standard filter BWC check to allow for cacheing bug (#62649) The `standard` tokenfilter was removed by #33310, and should have been unuseable in any indexes created since 7.0. However, a cacheing bug fixed by #51092 meant that it was still possible in certain circumstances to create indexes referencing the standard filter in versions up to 7.5.2. Our checks in AnalysisModule still refer to 7.0.0, however, meaning that a cluster that contains one of these rogue indexes cannot be upgraded. This commit adjusts the AnalysisModule checks so that we only refuse to build a mapping referring to standard filter if the index created version is 7.6 or later. Fixes #62644 --- .../indices/analysis/AnalysisModule.java | 5 +++- .../indices/analysis/AnalysisModuleTests.java | 23 +++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index 1e01474efc61b..c3b5bbf7d0c13 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -181,7 +181,10 @@ static Map setupPreConfiguredTokenFilters(List // Add "standard" for old indices (bwc) preConfiguredTokenFilters.register( "standard", PreConfiguredTokenFilter.elasticsearchVersion("standard", true, (reader, version) -> { - if (version.before(Version.V_7_0_0)) { + // This was originally removed in 7_0_0 but due to a cacheing bug it was still possible + // in certain circumstances to create a new index referencing the standard token filter + // until version 7_5_2 + if (version.before(Version.V_7_6_0)) { deprecationLogger.deprecate("standard_deprecation", "The [standard] token filter is deprecated and will be removed in a future version."); } else { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 483e9b1578ffe..efd3425948fc3 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -228,28 +228,27 @@ public void testUnderscoreInAnalyzerName() throws IOException { } public void testStandardFilterBWC() throws IOException { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT.minimumCompatibilityVersion()); - // bwc deprecation + // standard tokenfilter should have been removed entirely in the 7x line. However, a + // cacheing bug meant that it was still possible to create indexes using a standard + // filter until 7.6 { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_6_0, Version.CURRENT); final Settings settings = Settings.builder().put("index.analysis.analyzer.my_standard.tokenizer", "standard") .put("index.analysis.analyzer.my_standard.filter", "standard") .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(IndexMetadata.SETTING_VERSION_CREATED, version) .build(); - IndexAnalyzers analyzers = getIndexAnalyzers(settings); - assertTokenStreamContents(analyzers.get("my_standard").tokenStream("", "test"), new String[]{"test"}); - assertWarnings("The [standard] token filter is deprecated and will be removed in a future version."); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> getIndexAnalyzers(settings)); + assertThat(exc.getMessage(), equalTo("The [standard] token filter has been removed.")); } - // removal { - final Settings settings = Settings.builder() - .put("index.analysis.analyzer.my_standard.tokenizer", "standard") + Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_5_2); + final Settings settings = Settings.builder().put("index.analysis.analyzer.my_standard.tokenizer", "standard") .put("index.analysis.analyzer.my_standard.filter", "standard") - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_7_0_0) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).put(IndexMetadata.SETTING_VERSION_CREATED, version) .build(); - IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> getIndexAnalyzers(settings)); - assertThat(exc.getMessage(), equalTo("The [standard] token filter has been removed.")); + getIndexAnalyzers(settings); + assertWarnings("The [standard] token filter is deprecated and will be removed in a future version."); } }