From 750f67638ada1e3519438bc413c7f8f438f0db15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 2 Apr 2019 16:24:53 +0200 Subject: [PATCH 01/20] WIP --- .../common/SynonymTokenFilterFactory.java | 13 ++ .../analysis/common/SynonymAnalyzerIT.java | 127 +++++++++++++ .../analysis/common/SynonymAnalyzerTests.java | 112 +++++++++++ .../refresh/TransportShardRefreshAction.java | 12 ++ .../index/analysis/AnalysisRegistry.java | 178 ++++++++++++------ .../index/analysis/IndexAnalyzers.java | 28 ++- .../index/mapper/FieldTypeLookup.java | 12 ++ .../index/mapper/MapperService.java | 36 +++- 8 files changed, 456 insertions(+), 62 deletions(-) create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 75d4eca4254f8..5d6135549b882 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; +import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; @@ -50,6 +51,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { private final boolean lenient; protected final Settings settings; protected final Environment environment; + private final boolean updateable; SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { @@ -65,9 +67,15 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { this.expand = settings.getAsBoolean("expand", true); this.lenient = settings.getAsBoolean("lenient", false); this.format = settings.get("format", ""); + this.updateable = settings.getAsBoolean("updateable", false); this.environment = env; } + @Override + public AnalysisMode getAnalysisMode() { + return this.updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; + } + @Override public TokenStream create(TokenStream tokenStream) { throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first"); @@ -98,6 +106,11 @@ public TokenFilterFactory getSynonymFilter() { // which doesn't support stacked input tokens return IDENTITY_FILTER; } + + @Override + public AnalysisMode getAnalysisMode() { + return updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; + } }; } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java new file mode 100644 index 0000000000000..2c6d66720d390 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +@AwaitsFix(bugUrl="Cannot be run outside IDE yet") +public class SynonymAnalyzerIT extends ESIntegTestCase { + + private static Path config; + private static Path synonymsFile; + private static final String synonymsFileName = "synonyms.txt"; + + @BeforeClass + public static void initConfigDir() throws IOException { + config = createTempDir().resolve("config"); + if (Files.exists(config) == false) { + Files.createDirectory(config); + } + synonymsFile = config.resolve(synonymsFileName); + Files.createFile(synonymsFile); + assertTrue(Files.exists(synonymsFile)); + } + + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + @Override + protected Path nodeConfigPath(int nodeOrdinal) { + return config; + } + + public void testSynonymsUpdateable() throws FileNotFoundException, IOException { + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) { + out.println("foo, baz"); + } + assertTrue(Files.exists(synonymsFile)); + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") + .put("analysis.analyzer.my_synonym_analyzer.filter", "my_synonym_filter") + .put("analysis.filter.my_synonym_filter.type", "synonym") + .put("analysis.filter.my_synonym_filter.updateable", "true") + .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName)) + .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer")); + + client().prepareIndex("test", "_doc", "1").setSource("field", "foo").get(); + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 0L); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(2, analyzeResponse.getTokens().size()); + assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm()); + assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm()); + + // now update synonyms file and trigger reloading + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, buzz"); + } + // TODO don't use refresh here but something more specific + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(3, analyzeResponse.getTokens().size()); + Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains("buzz")); + + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 1L); + } +} \ No newline at end of file diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java new file mode 100644 index 0000000000000..9175999bb687b --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +public class SynonymAnalyzerTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + public void testSynonymsUpdateable() throws FileNotFoundException, IOException { + String synonymsFileName = "synonyms.txt"; + Path configDir = node().getEnvironment().configFile(); + if (Files.exists(configDir) == false) { + Files.createDirectory(configDir); + } + Path synonymsFile = configDir.resolve(synonymsFileName); + if (Files.exists(synonymsFile) == false) { + Files.createFile(synonymsFile); + } + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz"); + } + + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") + .putList("analysis.analyzer.my_synonym_analyzer.filter", "lowercase", "my_synonym_filter") + .put("analysis.filter.my_synonym_filter.type", "synonym") + .put("analysis.filter.my_synonym_filter.updateable", "true") + .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName)) + .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer")); + + client().prepareIndex("test", "_doc", "1").setSource("field", "Foo").get(); + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 0L); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(2, analyzeResponse.getTokens().size()); + assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm()); + assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm()); + + // now update synonyms file and trigger reloading + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, buzz"); + } + // TODO don't use refresh here but something more specific + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + analyzeResponse = client().admin().indices().prepareAnalyze("test", "Foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(3, analyzeResponse.getTokens().size()); + Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains("buzz")); + + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 1L); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index c0a52ac8c0d6a..2a30bd11e6566 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -58,6 +58,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind ActionListener> listener) { ActionListener.completeWith(listener, () -> { primary.refresh("api"); + try { + primary.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage(), ex); + return new PrimaryResult(null, null, ex); + } logger.trace("{} refresh request executed on primary", primary.shardId()); return new PrimaryResult<>(shardRequest, new ReplicationResponse()); }); @@ -66,6 +72,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind @Override protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica) { replica.refresh("api"); + try { + replica.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage(), ex); + return new ReplicaResult(ex); + } logger.trace("{} refresh request executed on replica", replica.shardId()); return new ReplicaResult(); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 483a1b4a7e563..674bca2ca2726 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -34,11 +33,14 @@ import java.io.Closeable; import java.io.IOException; +import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Collections.unmodifiableMap; @@ -151,13 +153,12 @@ public void close() throws IOException { * Creates an index-level {@link IndexAnalyzers} from this registry using the given index settings */ public IndexAnalyzers build(IndexSettings indexSettings) throws IOException { - final Map charFilterFactories = buildCharFilterFactories(indexSettings); final Map tokenizerFactories = buildTokenizerFactories(indexSettings); final Map tokenFilterFactories = buildTokenFilterFactories(indexSettings); - final Map> analyzierFactories = buildAnalyzerFactories(indexSettings); + final Map> analyzerFactories = buildAnalyzerFactories(indexSettings); final Map> normalizerFactories = buildNormalizerFactories(indexSettings); - return build(indexSettings, analyzierFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); + return build(indexSettings, analyzerFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); } public Map buildTokenFilterFactories(IndexSettings indexSettings) throws IOException { @@ -197,13 +198,21 @@ public Map> buildNormalizerFactories(IndexSettings i * @return {@link TokenizerFactory} provider or null */ public AnalysisProvider getTokenizerProvider(String tokenizer, IndexSettings indexSettings) { - final Map tokenizerSettings = indexSettings.getSettings().getGroups("index.analysis.tokenizer"); - if (tokenizerSettings.containsKey(tokenizer)) { - Settings currentSettings = tokenizerSettings.get(tokenizer); - return getAnalysisProvider(Component.TOKENIZER, tokenizers, tokenizer, currentSettings.get("type")); - } else { - return getTokenizerProvider(tokenizer); - } + return getProvider(Component.TOKENIZER, tokenizer, indexSettings, "index.analysis.tokenizer", tokenizers, + this::getTokenizerProvider); + } + + /** + * Returns a registered {@link CharFilterFactory} provider by {@link IndexSettings} + * or a registered {@link CharFilterFactory} provider by predefined name + * or null if the charFilter was not registered + * @param charFilter global or defined charFilter name + * @param indexSettings an index settings + * @return {@link CharFilterFactory} provider or null + */ + public AnalysisProvider getCharFilterProvider(String charFilter, IndexSettings indexSettings) { + return getProvider(Component.CHAR_FILTER, charFilter, indexSettings, "index.analysis.char_filter", charFilters, + this::getCharFilterProvider); } /** @@ -215,31 +224,18 @@ public AnalysisProvider getTokenizerProvider(String tokenizer, * @return {@link TokenFilterFactory} provider or null */ public AnalysisProvider getTokenFilterProvider(String tokenFilter, IndexSettings indexSettings) { - final Map tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.filter"); - if (tokenFilterSettings.containsKey(tokenFilter)) { - Settings currentSettings = tokenFilterSettings.get(tokenFilter); - String typeName = currentSettings.get("type"); - return getAnalysisProvider(Component.FILTER, tokenFilters, tokenFilter, typeName); - } else { - return getTokenFilterProvider(tokenFilter); - } + return getProvider(Component.FILTER, tokenFilter, indexSettings, "index.analysis.filter", tokenFilters, + this::getTokenFilterProvider); } - /** - * Returns a registered {@link CharFilterFactory} provider by {@link IndexSettings} - * or a registered {@link CharFilterFactory} provider by predefined name - * or null if the charFilter was not registered - * @param charFilter global or defined charFilter name - * @param indexSettings an index settings - * @return {@link CharFilterFactory} provider or null - */ - public AnalysisProvider getCharFilterProvider(String charFilter, IndexSettings indexSettings) { - final Map tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.char_filter"); - if (tokenFilterSettings.containsKey(charFilter)) { - Settings currentSettings = tokenFilterSettings.get(charFilter); - return getAnalysisProvider(Component.CHAR_FILTER, charFilters, charFilter, currentSettings.get("type")); + private AnalysisProvider getProvider(Component componentType, String componentName, IndexSettings indexSettings, + String componentSettings, Map> providers, Function> providerFunction) { + final Map subSettings = indexSettings.getSettings().getGroups(componentSettings); + if (subSettings.containsKey(componentName)) { + Settings currentSettings = subSettings.get(componentName); + return getAnalysisProvider(componentType, providers, componentName, currentSettings.get("type")); } else { - return getCharFilterProvider(charFilter); + return providerFunction.apply(componentName); } } @@ -277,7 +273,7 @@ public String toString() { } @SuppressWarnings("unchecked") - private Map buildMapping(Component component, IndexSettings settings, Map settingsMap, + Map buildMapping(Component component, IndexSettings settings, Map settingsMap, Map> providerMap, Map> defaultInstance) throws IOException { Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build(); @@ -318,9 +314,9 @@ private Map buildMapping(Component component, IndexSettings setti } // go over the char filters in the bindings and register the ones that are not configured - for (Map.Entry> entry : providerMap.entrySet()) { + for (Map.Entry> entry : providerMap.entrySet()) { String name = entry.getKey(); - AnalysisModule.AnalysisProvider provider = entry.getValue(); + AnalysisProvider provider = entry.getValue(); // we don't want to re-register one that already exists if (settingsMap.containsKey(name)) { continue; @@ -342,17 +338,12 @@ private Map buildMapping(Component component, IndexSettings setti for (Map.Entry> entry : defaultInstance.entrySet()) { final String name = entry.getKey(); final AnalysisModule.AnalysisProvider provider = entry.getValue(); - if (factories.containsKey(name) == false) { - final T instance = provider.get(settings, environment, name, defaultSettings); - if (factories.containsKey(name) == false) { - factories.put(name, instance); - } - } + factories.putIfAbsent(name, provider.get(settings, environment, name, defaultSettings)); } return factories; } - private AnalysisProvider getAnalysisProvider(Component component, Map> providerMap, + private static AnalysisProvider getAnalysisProvider(Component component, Map> providerMap, String name, String typeName) { if (typeName == null) { throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer"); @@ -364,9 +355,9 @@ private AnalysisProvider getAnalysisProvider(Component component, Map>> analyzerProviderFactories; + final Map>> analyzerProviderFactories; final Map> preConfiguredTokenFilters; final Map> preConfiguredTokenizers; final Map> preConfiguredCharFilterFactories; @@ -391,19 +382,19 @@ private PrebuiltAnalysis( this.preConfiguredTokenizers = preConfiguredTokenizers; } - public AnalysisModule.AnalysisProvider getCharFilterFactory(String name) { + public AnalysisProvider getCharFilterFactory(String name) { return preConfiguredCharFilterFactories.get(name); } - public AnalysisModule.AnalysisProvider getTokenFilterFactory(String name) { + public AnalysisProvider getTokenFilterFactory(String name) { return preConfiguredTokenFilters.get(name); } - public AnalysisModule.AnalysisProvider getTokenizerFactory(String name) { + public AnalysisProvider getTokenizerFactory(String name) { return preConfiguredTokenizers.get(name); } - public AnalysisModule.AnalysisProvider> getAnalyzerProvider(String name) { + public AnalysisProvider> getAnalyzerProvider(String name) { return analyzerProviderFactories.get(name); } @@ -420,9 +411,6 @@ public IndexAnalyzers build(IndexSettings indexSettings, Map tokenizerFactoryFactories, Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { - - Index index = indexSettings.getIndex(); - analyzerProviders = new HashMap<>(analyzerProviders); Map analyzers = new HashMap<>(); Map normalizers = new HashMap<>(); Map whitespaceNormalizers = new HashMap<>(); @@ -458,7 +446,7 @@ public IndexAnalyzers build(IndexSettings indexSettings, defaultAnalyzer.checkAllowedInMode(AnalysisMode.ALL); if (analyzers.containsKey("default_index")) { throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use " + - "[index.analysis.analyzer.default] instead for index [" + index.getName() + "]"); + "[index.analysis.analyzer.default] instead for index [" + indexSettings.getIndex().getName() + "]"); } NamedAnalyzer defaultSearchAnalyzer = analyzers.getOrDefault("default_search", defaultAnalyzer); NamedAnalyzer defaultSearchQuoteAnalyzer = analyzers.getOrDefault("default_search_quote", defaultSearchAnalyzer); @@ -472,9 +460,11 @@ public IndexAnalyzers build(IndexSettings indexSettings, whitespaceNormalizers); } - private static NamedAnalyzer produceAnalyzer(String name, AnalyzerProvider analyzerFactory, - Map tokenFilters, Map charFilters, - Map tokenizers) { + private static NamedAnalyzer produceAnalyzer(String name, + AnalyzerProvider analyzerFactory, + Map tokenFilters, + Map charFilters, + Map tokenizers) { /* * Lucene defaults positionIncrementGap to 0 in all analyzers but * Elasticsearch defaults them to 0 only before version 2.0 @@ -536,4 +526,80 @@ private void processNormalizerFactory( } normalizers.put(name, normalizer); } + + /** + * Create an new IndexAnalyzer instance based on the existing one. Analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried to + * be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. + */ + public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { + NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexSettings); + NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexSettings); + Map newAnalyzers = new HashMap<>(); + for (NamedAnalyzer analyzer : indexAnalyzers.getAnalyzers().values()) { + newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexSettings)); + } + return new IndexAnalyzers(indexSettings, indexAnalyzers.getDefaultIndexAnalyzer(), newDefaultSearchAnalyzer, + newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers()); + } + + /** + * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider + * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. + */ + private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings) + throws IOException { + // only rebuild custom analyzers that are in SEARCH_TIME mode + if ((oldAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == false + || (oldAnalyzer.analyzer() instanceof CustomAnalyzer == false)) { + return oldAnalyzer; + } else { + String analyzerName = oldAnalyzer.name(); + + // get tokenizer necessary to re-build the analyzer + String tokenizer = indexSettings.getSettings().get("index.analysis.analyzer." + analyzerName + ".tokenizer"); + Map> tokenizerProvider = Collections.singletonMap(tokenizer, + getTokenizerProvider(tokenizer, indexSettings)); + final Map tokenizerSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER); + final Map tokenizers = buildMapping(Component.TOKENIZER, indexSettings, tokenizerSettings, + tokenizerProvider, Collections.emptyMap()); + + // get char filters necessary to re-build the analyzer + List charFilterNames = indexSettings.getSettings() + .getAsList("index.analysis.analyzer." + analyzerName + ".char_filter"); + Map> charFilterProvider = charFilterNames.stream() + .map(s -> new AbstractMap.SimpleEntry<>(s, getCharFilterProvider(s, indexSettings))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER); + final Map charFilters = buildMapping(Component.CHAR_FILTER, indexSettings, + charFiltersSettings, charFilterProvider, Collections.emptyMap()); + + // get token filters necessary to re-build the analyzer + List tokenFilterNames = indexSettings.getSettings().getAsList("index.analysis.analyzer." + analyzerName + ".filter"); + final Map tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER); + Map> tokenFilterProviders = new HashMap<>(); + for (String filterName : tokenFilterNames) { + AnalysisProvider tokenFilterProvider = getTokenFilterProvider(filterName, indexSettings); + Settings settings = tokenFiltersSettings.get(filterName); + if (settings != null) { + String type = settings.get("type"); + if (type != null) { + AnalysisProvider provider = getTokenFilterProvider(type, indexSettings); + tokenFilterProviders.put(type, provider); + } + } else { + tokenFilterProviders.put(filterName, tokenFilterProvider); + } + } + + final Map tokenFilters = buildMapping(Component.FILTER, indexSettings, + tokenFiltersSettings, tokenFilterProviders, prebuiltAnalysis.preConfiguredTokenFilters); + + Settings analyzerSettings = indexSettings.getSettings().getAsSettings("index.analysis.analyzer." + analyzerName); + AnalyzerProvider analyzerProvider = new CustomAnalyzerProvider(indexSettings, analyzerName, analyzerSettings); + + // produce the analyzer + return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, tokenFilters, charFilters, tokenizers); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 4cb0b9aa324c9..0ebf7360f2c3a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -24,6 +24,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -54,9 +55,9 @@ public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAna this.defaultIndexAnalyzer = defaultIndexAnalyzer; this.defaultSearchAnalyzer = defaultSearchAnalyzer; this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer; - this.analyzers = unmodifiableMap(analyzers); - this.normalizers = unmodifiableMap(normalizers); - this.whitespaceNormalizers = unmodifiableMap(whitespaceNormalizers); + this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); + this.normalizers = unmodifiableMap(new HashMap<>(normalizers)); + this.whitespaceNormalizers = unmodifiableMap(new HashMap<>(whitespaceNormalizers)); } /** @@ -66,6 +67,13 @@ public NamedAnalyzer get(String name) { return analyzers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index analyzers + */ + Map getAnalyzers() { + return analyzers; + } + /** * Returns a normalizer mapped to the given name or null if not present */ @@ -73,6 +81,13 @@ public NamedAnalyzer getNormalizer(String name) { return normalizers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index normalizers + */ + Map getNormalizers() { + return normalizers; + } + /** * Returns a normalizer that splits on whitespace mapped to the given name or null if not present */ @@ -80,6 +95,13 @@ public NamedAnalyzer getWhitespaceNormalizer(String name) { return whitespaceNormalizers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index whitespace normalizers + */ + Map getWhitespaceNormalizers() { + return whitespaceNormalizers; + } + /** * Returns the default index analyzer for this index */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 27d061d8c2788..75ebc9eecc466 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -86,6 +86,18 @@ public FieldTypeLookup copyAndAddAll(String type, return new FieldTypeLookup(fullName, aliases); } + /** + * Return a new instance that contains the union of this instance and the mapped field types + */ + public FieldTypeLookup copyAndAddAll(Collection mappedFieldTypes) { + CopyOnWriteHashMap fullName = this.fullNameToFieldType; + CopyOnWriteHashMap aliases = this.aliasToConcreteName; + + for (MappedFieldType mft : mappedFieldTypes) { + fullName = fullName.copyAndPut(mft.name(), mft); + } + return new FieldTypeLookup(fullName, aliases); + } /** Returns the field for the given field */ public MappedFieldType get(String field) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 487a6ac4789e3..fba9220eb703a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.analysis.AnalysisMode; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.Mapper.BuilderContext; @@ -70,7 +72,9 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -122,7 +126,7 @@ public enum MergeReason { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(MapperService.class)); - private final IndexAnalyzers indexAnalyzers; + private IndexAnalyzers indexAnalyzers; private volatile String defaultMappingSource; @@ -136,8 +140,8 @@ public enum MergeReason { private final DocumentMapperParser documentParser; private final MapperAnalyzerWrapper indexAnalyzer; - private final MapperAnalyzerWrapper searchAnalyzer; - private final MapperAnalyzerWrapper searchQuoteAnalyzer; + private MapperAnalyzerWrapper searchAnalyzer; + private MapperAnalyzerWrapper searchQuoteAnalyzer; private volatile Map unmappedFieldTypes = emptyMap(); @@ -843,4 +847,30 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { return defaultAnalyzer; } } + + public void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { + logger.info("reloading search analyzers"); + + // refresh indexAnalyzers and search analyzers + this.indexAnalyzers = registry.rebuildIndexAnalyzers(this.indexAnalyzers, indexSettings); + this.searchAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); + this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchQuoteAnalyzer(), + p -> p.searchQuoteAnalyzer()); + + // also reload search time analyzers in MappedFieldTypes + // refresh search time analyzers in MappedFieldTypes + List mftsToRefresh = StreamSupport.stream(fieldTypes.spliterator(), false) + .filter(mft -> (mft.searchAnalyzer() != null && mft.searchAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME) + || (mft.searchQuoteAnalyzer() != null && mft.searchQuoteAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME)) + .collect(Collectors.toList()); + List updated = mftsToRefresh.stream().map(mft -> { + MappedFieldType newMft = mft.clone(); + newMft.setSearchAnalyzer(indexAnalyzers.get(mft.searchAnalyzer().name())); + newMft.setSearchQuoteAnalyzer(indexAnalyzers.get(mft.searchQuoteAnalyzer().name())); + newMft.freeze(); + return newMft; + }).collect(Collectors.toList()); + fieldTypes = fieldTypes.copyAndAddAll(updated); + // mapper.root().updateFieldType(); + } } From 2a7951ba104b5697e935ffa2f62915325b6f519d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 Apr 2019 12:37:02 +0200 Subject: [PATCH 02/20] adressing comments --- .../elasticsearch/index/analysis/AnalysisRegistry.java | 2 +- .../org/elasticsearch/index/mapper/MapperService.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 674bca2ca2726..06a6986ebfbcc 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -355,7 +355,7 @@ private static AnalysisProvider getAnalysisProvider(Component component, return type; } - static class PrebuiltAnalysis implements Closeable { + private static class PrebuiltAnalysis implements Closeable { final Map>> analyzerProviderFactories; final Map> preConfiguredTokenFilters; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index fba9220eb703a..4b8351cbd287d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -126,7 +126,7 @@ public enum MergeReason { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(MapperService.class)); - private IndexAnalyzers indexAnalyzers; + private volatile IndexAnalyzers indexAnalyzers; private volatile String defaultMappingSource; @@ -140,8 +140,8 @@ public enum MergeReason { private final DocumentMapperParser documentParser; private final MapperAnalyzerWrapper indexAnalyzer; - private MapperAnalyzerWrapper searchAnalyzer; - private MapperAnalyzerWrapper searchQuoteAnalyzer; + private volatile MapperAnalyzerWrapper searchAnalyzer; + private volatile MapperAnalyzerWrapper searchQuoteAnalyzer; private volatile Map unmappedFieldTypes = emptyMap(); @@ -848,7 +848,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { } } - public void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { + public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { logger.info("reloading search analyzers"); // refresh indexAnalyzers and search analyzers @@ -871,6 +871,6 @@ public void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException return newMft; }).collect(Collectors.toList()); fieldTypes = fieldTypes.copyAndAddAll(updated); - // mapper.root().updateFieldType(); + mapper.root().updateFieldType(fieldTypes.fullNameToFieldType); } } From 07b81560f1b6121619106ea470d1a5aaaa4a8495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 15 Apr 2019 14:43:31 +0200 Subject: [PATCH 03/20] Access analysis providers via IndexAnalyzers --- .../index/analysis/AnalysisRegistry.java | 65 ++++--------------- .../index/analysis/IndexAnalyzers.java | 59 ++++++++++++++++- .../index/mapper/MapperService.java | 4 +- 3 files changed, 75 insertions(+), 53 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 06a6986ebfbcc..45bdec0a8af5e 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -26,6 +26,7 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.IndexAnalyzers.IndexAnalysisProviders; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; @@ -33,10 +34,8 @@ import java.io.Closeable; import java.io.IOException; -import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -456,8 +455,10 @@ public IndexAnalyzers build(IndexSettings indexSettings, throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\""); } } + IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(analyzerProviders, normalizerProviders, + tokenizerFactoryFactories, charFilterFactoryFactories, tokenFilterFactoryFactories); return new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer, analyzers, normalizers, - whitespaceNormalizers); + whitespaceNormalizers, analysisProviders); } private static NamedAnalyzer produceAnalyzer(String name, @@ -532,21 +533,24 @@ private void processNormalizerFactory( * be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. */ public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { - NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexSettings); - NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexSettings); + NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexAnalyzers, + indexSettings); + NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexAnalyzers, + indexSettings); Map newAnalyzers = new HashMap<>(); for (NamedAnalyzer analyzer : indexAnalyzers.getAnalyzers().values()) { - newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexSettings)); + newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexAnalyzers, indexSettings)); } return new IndexAnalyzers(indexSettings, indexAnalyzers.getDefaultIndexAnalyzer(), newDefaultSearchAnalyzer, - newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers()); + newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers(), + indexAnalyzers.getAnalysisProviders()); } /** * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. */ - private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings) + private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { // only rebuild custom analyzers that are in SEARCH_TIME mode if ((oldAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == false @@ -554,51 +558,10 @@ private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSetting return oldAnalyzer; } else { String analyzerName = oldAnalyzer.name(); - - // get tokenizer necessary to re-build the analyzer - String tokenizer = indexSettings.getSettings().get("index.analysis.analyzer." + analyzerName + ".tokenizer"); - Map> tokenizerProvider = Collections.singletonMap(tokenizer, - getTokenizerProvider(tokenizer, indexSettings)); - final Map tokenizerSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER); - final Map tokenizers = buildMapping(Component.TOKENIZER, indexSettings, tokenizerSettings, - tokenizerProvider, Collections.emptyMap()); - - // get char filters necessary to re-build the analyzer - List charFilterNames = indexSettings.getSettings() - .getAsList("index.analysis.analyzer." + analyzerName + ".char_filter"); - Map> charFilterProvider = charFilterNames.stream() - .map(s -> new AbstractMap.SimpleEntry<>(s, getCharFilterProvider(s, indexSettings))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final Map charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER); - final Map charFilters = buildMapping(Component.CHAR_FILTER, indexSettings, - charFiltersSettings, charFilterProvider, Collections.emptyMap()); - - // get token filters necessary to re-build the analyzer - List tokenFilterNames = indexSettings.getSettings().getAsList("index.analysis.analyzer." + analyzerName + ".filter"); - final Map tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER); - Map> tokenFilterProviders = new HashMap<>(); - for (String filterName : tokenFilterNames) { - AnalysisProvider tokenFilterProvider = getTokenFilterProvider(filterName, indexSettings); - Settings settings = tokenFiltersSettings.get(filterName); - if (settings != null) { - String type = settings.get("type"); - if (type != null) { - AnalysisProvider provider = getTokenFilterProvider(type, indexSettings); - tokenFilterProviders.put(type, provider); - } - } else { - tokenFilterProviders.put(filterName, tokenFilterProvider); - } - } - - final Map tokenFilters = buildMapping(Component.FILTER, indexSettings, - tokenFiltersSettings, tokenFilterProviders, prebuiltAnalysis.preConfiguredTokenFilters); - Settings analyzerSettings = indexSettings.getSettings().getAsSettings("index.analysis.analyzer." + analyzerName); AnalyzerProvider analyzerProvider = new CustomAnalyzerProvider(indexSettings, analyzerName, analyzerSettings); - - // produce the analyzer - return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, tokenFilters, charFilters, tokenizers); + return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, indexAnalyzers.getTokenFilterFactoryFactories(), + indexAnalyzers.getCharFilterFactoryFactories(), indexAnalyzers.getTokenizerFactoryFactories()); } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 0ebf7360f2c3a..3fe0d4967699b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -24,6 +24,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -44,10 +45,19 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos private final Map analyzers; private final Map normalizers; private final Map whitespaceNormalizers; + private final IndexAnalysisProviders analysisProviders; + + public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAnalyzer, NamedAnalyzer defaultSearchAnalyzer, + NamedAnalyzer defaultSearchQuoteAnalyzer, Map analyzers, Map normalizers, + Map whitespaceNormalizers) { + this(indexSettings, defaultIndexAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer, analyzers, normalizers, + whitespaceNormalizers, IndexAnalysisProviders.EMPTY); + } public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAnalyzer, NamedAnalyzer defaultSearchAnalyzer, NamedAnalyzer defaultSearchQuoteAnalyzer, Map analyzers, - Map normalizers, Map whitespaceNormalizers) { + Map normalizers, Map whitespaceNormalizers, + IndexAnalysisProviders analysisProviders) { super(indexSettings); if (defaultIndexAnalyzer.name().equals("default") == false) { throw new IllegalStateException("default analyzer must have the name [default] but was: [" + defaultIndexAnalyzer.name() + "]"); @@ -58,6 +68,7 @@ public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAna this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); this.normalizers = unmodifiableMap(new HashMap<>(normalizers)); this.whitespaceNormalizers = unmodifiableMap(new HashMap<>(whitespaceNormalizers)); + this.analysisProviders = analysisProviders; } /** @@ -123,10 +134,56 @@ public NamedAnalyzer getDefaultSearchQuoteAnalyzer() { return defaultSearchQuoteAnalyzer; } + public IndexAnalysisProviders getAnalysisProviders() { + return analysisProviders; + } + + public Map> getAnalyzerProviders() { + return analysisProviders.analyzerProviders; + } + + public Map> getNormalizerProviders() { + return analysisProviders.normalizerProviders; + } + + public Map getTokenizerFactoryFactories() { + return analysisProviders.tokenizerFactoryFactories; + } + + public Map getCharFilterFactoryFactories() { + return analysisProviders.charFilterFactoryFactories; + } + + public Map getTokenFilterFactoryFactories() { + return analysisProviders.tokenFilterFactoryFactories; + } + @Override public void close() throws IOException { IOUtils.close(() -> Stream.concat(analyzers.values().stream(), normalizers.values().stream()) .filter(a -> a.scope() == AnalyzerScope.INDEX) .iterator()); } + + static class IndexAnalysisProviders { + + static final IndexAnalysisProviders EMPTY = new IndexAnalysisProviders(Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + + private final Map> analyzerProviders; + private final Map> normalizerProviders; + private final Map tokenizerFactoryFactories; + private final Map charFilterFactoryFactories; + private final Map tokenFilterFactoryFactories; + + IndexAnalysisProviders(Map> analyzerProviders, + Map> normalizerProviders, Map tokenizerFactoryFactories, + Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { + this.analyzerProviders = unmodifiableMap(analyzerProviders); + this.normalizerProviders = unmodifiableMap(normalizerProviders); + this.tokenizerFactoryFactories = unmodifiableMap(tokenizerFactoryFactories); + this.charFilterFactoryFactories = unmodifiableMap(charFilterFactoryFactories); + this.tokenFilterFactoryFactories = unmodifiableMap(tokenFilterFactoryFactories); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 4b8351cbd287d..70de937d6ed77 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -871,6 +871,8 @@ public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) th return newMft; }).collect(Collectors.toList()); fieldTypes = fieldTypes.copyAndAddAll(updated); - mapper.root().updateFieldType(fieldTypes.fullNameToFieldType); + if (mapper != null) { + mapper.root().updateFieldType(fieldTypes.fullNameToFieldType); + } } } From 3de3ec58ab6b82ae32303843966f0e82f787996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 17 Apr 2019 17:46:54 +0200 Subject: [PATCH 04/20] Improve reloading token filters --- .../index/analysis/AnalysisRegistry.java | 78 +++++++++++++++---- .../index/analysis/IndexAnalyzers.java | 21 +---- 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 45bdec0a8af5e..3e9c78bb6ecb7 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -34,13 +34,19 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.unmodifiableMap; @@ -162,8 +168,8 @@ public IndexAnalyzers build(IndexSettings indexSettings) throws IOException { public Map buildTokenFilterFactories(IndexSettings indexSettings) throws IOException { final Map tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER); - return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, - Collections.unmodifiableMap(this.tokenFilters), prebuiltAnalysis.preConfiguredTokenFilters); + return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, this.tokenFilters, + prebuiltAnalysis.preConfiguredTokenFilters); } public Map buildTokenizerFactories(IndexSettings indexSettings) throws IOException { @@ -455,8 +461,8 @@ public IndexAnalyzers build(IndexSettings indexSettings, throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\""); } } - IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(analyzerProviders, normalizerProviders, - tokenizerFactoryFactories, charFilterFactoryFactories, tokenFilterFactoryFactories); + IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(tokenizerFactoryFactories, charFilterFactoryFactories, + tokenFilterFactoryFactories); return new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer, analyzers, normalizers, whitespaceNormalizers, analysisProviders); } @@ -533,25 +539,68 @@ private void processNormalizerFactory( * be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. */ public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { - NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexAnalyzers, - indexSettings); - NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexAnalyzers, - indexSettings); + + // scan analyzers to collect token filters that we need to reload + List analyzers = Stream.concat( + Stream.of(indexAnalyzers.getDefaultSearchAnalyzer(), indexAnalyzers.getDefaultIndexAnalyzer()), + indexAnalyzers.getAnalyzers().values().stream()).collect(Collectors.toList()); + + Set filtersThatNeedReloading = filtersThatNeedReloading(analyzers); + final Map tokenFiltersToReloading = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER) + .entrySet().stream() + .filter(entry -> filtersThatNeedReloading.contains(entry.getKey())) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + + final Map newTokenFilterFactories = buildMapping(Component.FILTER, indexSettings, + tokenFiltersToReloading, this.tokenFilters, prebuiltAnalysis.preConfiguredTokenFilters); + + // fill the rest of the token filter factory map with the entries that are missing (were not reloaded) + for (Entry entry : indexAnalyzers.getTokenFilterFactoryFactories().entrySet()) { + newTokenFilterFactories.putIfAbsent(entry.getKey(), entry.getValue()); + } + + // char filters and tokenizers are not updateable, so we can use the old ones + Map currentCharFilterFactories = indexAnalyzers.getCharFilterFactoryFactories(); + Map currentTokenizerFactories = indexAnalyzers.getTokenizerFactoryFactories(); + NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexSettings, + currentCharFilterFactories, currentTokenizerFactories, newTokenFilterFactories); + NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexSettings, + currentCharFilterFactories, currentTokenizerFactories, newTokenFilterFactories); Map newAnalyzers = new HashMap<>(); for (NamedAnalyzer analyzer : indexAnalyzers.getAnalyzers().values()) { - newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexAnalyzers, indexSettings)); + newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexSettings, currentCharFilterFactories, + currentTokenizerFactories, newTokenFilterFactories)); } + + IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(currentTokenizerFactories, currentCharFilterFactories, + newTokenFilterFactories); return new IndexAnalyzers(indexSettings, indexAnalyzers.getDefaultIndexAnalyzer(), newDefaultSearchAnalyzer, newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers(), - indexAnalyzers.getAnalysisProviders()); + analysisProviders); + } + + private Set filtersThatNeedReloading(List analyzers) { + Set filters = new HashSet<>(); + for (NamedAnalyzer namedAnalyzer : analyzers) { + // only rebuild custom analyzers that are in SEARCH_TIME mode + if ((namedAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == true + && (namedAnalyzer.analyzer() instanceof CustomAnalyzer == true)) { + TokenFilterFactory[] currentTokenFilters = ((CustomAnalyzer) namedAnalyzer.analyzer()).tokenFilters(); + // get token filters necessary to re-build the analyzer + Arrays.stream(currentTokenFilters).filter(f -> f.getAnalysisMode() == AnalysisMode.SEARCH_TIME).map(f -> f.name()) + .collect(Collectors.toCollection(() -> filters)); + } + } + return filters; } /** * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. */ - private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) - throws IOException { + private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings, + Map charFilterFactories, Map tokenizerFactories, + Map tokenFilterFactories) throws IOException { // only rebuild custom analyzers that are in SEARCH_TIME mode if ((oldAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == false || (oldAnalyzer.analyzer() instanceof CustomAnalyzer == false)) { @@ -559,9 +608,10 @@ private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexAnalyze } else { String analyzerName = oldAnalyzer.name(); Settings analyzerSettings = indexSettings.getSettings().getAsSettings("index.analysis.analyzer." + analyzerName); + AnalyzerProvider analyzerProvider = new CustomAnalyzerProvider(indexSettings, analyzerName, analyzerSettings); - return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, indexAnalyzers.getTokenFilterFactoryFactories(), - indexAnalyzers.getCharFilterFactoryFactories(), indexAnalyzers.getTokenizerFactoryFactories()); + return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, tokenFilterFactories, charFilterFactories, + tokenizerFactories); } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 3fe0d4967699b..af0b8bb2c0d3d 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -134,18 +134,6 @@ public NamedAnalyzer getDefaultSearchQuoteAnalyzer() { return defaultSearchQuoteAnalyzer; } - public IndexAnalysisProviders getAnalysisProviders() { - return analysisProviders; - } - - public Map> getAnalyzerProviders() { - return analysisProviders.analyzerProviders; - } - - public Map> getNormalizerProviders() { - return analysisProviders.normalizerProviders; - } - public Map getTokenizerFactoryFactories() { return analysisProviders.tokenizerFactoryFactories; } @@ -168,19 +156,14 @@ public void close() throws IOException { static class IndexAnalysisProviders { static final IndexAnalysisProviders EMPTY = new IndexAnalysisProviders(Collections.emptyMap(), Collections.emptyMap(), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); + Collections.emptyMap()); - private final Map> analyzerProviders; - private final Map> normalizerProviders; private final Map tokenizerFactoryFactories; private final Map charFilterFactoryFactories; private final Map tokenFilterFactoryFactories; - IndexAnalysisProviders(Map> analyzerProviders, - Map> normalizerProviders, Map tokenizerFactoryFactories, + IndexAnalysisProviders(Map tokenizerFactoryFactories, Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { - this.analyzerProviders = unmodifiableMap(analyzerProviders); - this.normalizerProviders = unmodifiableMap(normalizerProviders); this.tokenizerFactoryFactories = unmodifiableMap(tokenizerFactoryFactories); this.charFilterFactoryFactories = unmodifiableMap(charFilterFactoryFactories); this.tokenFilterFactoryFactories = unmodifiableMap(tokenFilterFactoryFactories); From 5daca94255bca5d28a80b2d19742772e72ebe473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 18 Apr 2019 19:16:13 +0200 Subject: [PATCH 05/20] Adding AnalysisRegistry unit tests --- .../index/analysis/AnalysisRegistry.java | 4 +- .../index/analysis/AnalysisRegistryTests.java | 122 ++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 3e9c78bb6ecb7..e9d9d37b82e97 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -579,7 +579,7 @@ public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, Index analysisProviders); } - private Set filtersThatNeedReloading(List analyzers) { + static Set filtersThatNeedReloading(List analyzers) { Set filters = new HashSet<>(); for (NamedAnalyzer namedAnalyzer : analyzers) { // only rebuild custom analyzers that are in SEARCH_TIME mode @@ -598,7 +598,7 @@ private Set filtersThatNeedReloading(List analyzers) { * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. */ - private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings, + static NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings, Map charFilterFactories, Map tokenizerFactories, Map tokenFilterFactories) throws IOException { // only rebuild custom analyzers that are in SEARCH_TIME mode diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index b836a5d0372b8..6c103c0f7e062 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -43,8 +44,12 @@ import org.elasticsearch.test.VersionUtils; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -264,4 +269,121 @@ public void testEnsureCloseInvocationProperlyDelegated() throws IOException { registry.close(); verify(mock).close(); } + + /** + * test helper method that filters list of input analyzers to get the names of the token filters they contain + * that can be reloaded at search time + */ + public void testFiltersThatNeedReloading() { + TokenFilterFactory[] tokenFilters = new TokenFilterFactory[] { + createTokenFilter("first", AnalysisMode.ALL), + createTokenFilter("second", AnalysisMode.SEARCH_TIME), + createTokenFilter("third", AnalysisMode.ALL) + }; + List analyzers = Arrays.asList( + new NamedAnalyzer("myAnalyzer", AnalyzerScope.INDEX, new CustomAnalyzer("tokenizer", null, null, tokenFilters )), + new NamedAnalyzer("myAnalyzer", AnalyzerScope.INDEX, new StandardAnalyzer())); + Set filtersThatNeedReloading = AnalysisRegistry.filtersThatNeedReloading(analyzers); + assertEquals(1, filtersThatNeedReloading.size()); + assertTrue(filtersThatNeedReloading.contains("second")); + } + + private TokenFilterFactory createTokenFilter(String name, AnalysisMode mode) { + return new TokenFilterFactory() { + + @Override + public String name() { + return name; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return null; + } + + @Override + public AnalysisMode getAnalysisMode() { + return mode; + } + }; + } + + /** + * test helper function that rebuilds an input {@link NamedAnalyzer} if it is reloadable + * @throws IOException + */ + public void testRebuildIfNecessary() throws IOException { + NamedAnalyzer noReloading = new NamedAnalyzer("noReloading", AnalyzerScope.INDEX, new StandardAnalyzer()); + assertSame(noReloading, AnalysisRegistry.rebuildIfNecessary(noReloading, null, null, null, null)); + + TokenFilterFactory[] tokenFilters = new TokenFilterFactory[] { + createTokenFilter("first", AnalysisMode.INDEX_TIME), + createTokenFilter("second", AnalysisMode.ALL), + createTokenFilter("third", AnalysisMode.ALL) + }; + NamedAnalyzer noReloadingEither = new NamedAnalyzer("noReloadingEither", AnalyzerScope.INDEX, + new CustomAnalyzer("tokenizer", null, null, tokenFilters)); + assertSame(noReloadingEither, AnalysisRegistry.rebuildIfNecessary(noReloadingEither, null, null, null, null)); + + + Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") + .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); + + final AtomicInteger factoryCounter = new AtomicInteger(0); + TestAnalysis testAnalysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisPlugin() { + + @Override + public Map> getTokenFilters() { + return Collections.singletonMap("myReloadableFilter", new AnalysisProvider() { + + @Override + public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) + throws IOException { + factoryCounter.getAndIncrement(); + return new MyReloadableFilter(); + } + }); + } + }); + + tokenFilters[0] = testAnalysis.tokenFilter.get("myReloadableFilter"); + NamedAnalyzer reloadableAnalyzer = new NamedAnalyzer("reloadableAnalyzer", AnalyzerScope.INDEX, + new CustomAnalyzer("tokenizer", null, null, tokenFilters)); + IndexSettings indexSetings = new IndexSettings(IndexMetaData.builder("testIndex").settings(indexSettings).build(), indexSettings); + + NamedAnalyzer rebuilt = AnalysisRegistry.rebuildIfNecessary(reloadableAnalyzer, indexSetings, testAnalysis.charFilter, + testAnalysis.tokenizer, testAnalysis.tokenFilter); + assertEquals(reloadableAnalyzer.name(), rebuilt.name()); + assertNotSame(reloadableAnalyzer, rebuilt); + assertEquals(2, factoryCounter.get()); // once on intialization, once again for reloading + TokenFilterFactory reloadedFactory = ((CustomAnalyzer) rebuilt.analyzer()).tokenFilters()[0]; + assertThat(reloadedFactory, instanceOf(MyReloadableFilter.class)); + assertEquals(2, MyReloadableFilter.constructorCounter); + } + + static class MyReloadableFilter implements TokenFilterFactory { + + static int constructorCounter = 0; + + MyReloadableFilter() { + constructorCounter++; + } + + @Override + public String name() { + return "myReloadableFilter"; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return null; + } + @Override + public AnalysisMode getAnalysisMode() { + return AnalysisMode.SEARCH_TIME; + } + }; } From 97f1c58bdee0a2ccdb5301db17f1177751a1e6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 23 Apr 2019 15:30:52 +0200 Subject: [PATCH 06/20] Fix small compile issue --- .../org/elasticsearch/index/analysis/AnalysisRegistryTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 6c103c0f7e062..845446de7786b 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -310,7 +310,6 @@ public AnalysisMode getAnalysisMode() { /** * test helper function that rebuilds an input {@link NamedAnalyzer} if it is reloadable - * @throws IOException */ public void testRebuildIfNecessary() throws IOException { NamedAnalyzer noReloading = new NamedAnalyzer("noReloading", AnalyzerScope.INDEX, new StandardAnalyzer()); From 9aa1f28dbd32e39f80bfb2943ddf0240970a555b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 23 Apr 2019 17:14:38 +0200 Subject: [PATCH 07/20] Add unit test for AnalysisRegistry#reloadIndexAnalyzers --- .../index/analysis/AnalysisRegistry.java | 32 +++++---- .../index/mapper/MapperService.java | 2 +- .../index/analysis/AnalysisRegistryTests.java | 70 ++++++++++++++++--- .../org/elasticsearch/test/ESTestCase.java | 1 + 4 files changed, 80 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index e9d9d37b82e97..41f295571cc9b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -34,6 +34,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -46,7 +47,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.unmodifiableMap; @@ -535,24 +535,27 @@ private void processNormalizerFactory( } /** - * Create an new IndexAnalyzer instance based on the existing one. Analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried to - * be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. + * Creates an new IndexAnalyzer instance based on the existing one passed in. If there are no analysis components that need reloading, + * the same instance of {@link IndexAnalyzers} is returned. Otherwise, analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried + * to be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. */ - public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { - + public IndexAnalyzers reloadIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { // scan analyzers to collect token filters that we need to reload - List analyzers = Stream.concat( - Stream.of(indexAnalyzers.getDefaultSearchAnalyzer(), indexAnalyzers.getDefaultIndexAnalyzer()), - indexAnalyzers.getAnalyzers().values().stream()).collect(Collectors.toList()); - + Map oldAnalyzers = indexAnalyzers.getAnalyzers(); + List analyzers = new ArrayList<>(oldAnalyzers.values()); + analyzers.add(indexAnalyzers.getDefaultIndexAnalyzer()); + analyzers.add(indexAnalyzers.getDefaultSearchAnalyzer()); Set filtersThatNeedReloading = filtersThatNeedReloading(analyzers); - final Map tokenFiltersToReloading = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER) + if (filtersThatNeedReloading.size() == 0) { + return indexAnalyzers; + } + final Map tokenFiltersToReload = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER) .entrySet().stream() .filter(entry -> filtersThatNeedReloading.contains(entry.getKey())) .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); final Map newTokenFilterFactories = buildMapping(Component.FILTER, indexSettings, - tokenFiltersToReloading, this.tokenFilters, prebuiltAnalysis.preConfiguredTokenFilters); + tokenFiltersToReload, this.tokenFilters, prebuiltAnalysis.preConfiguredTokenFilters); // fill the rest of the token filter factory map with the entries that are missing (were not reloaded) for (Entry entry : indexAnalyzers.getTokenFilterFactoryFactories().entrySet()) { @@ -567,9 +570,10 @@ public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, Index NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexSettings, currentCharFilterFactories, currentTokenizerFactories, newTokenFilterFactories); Map newAnalyzers = new HashMap<>(); - for (NamedAnalyzer analyzer : indexAnalyzers.getAnalyzers().values()) { - newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexSettings, currentCharFilterFactories, - currentTokenizerFactories, newTokenFilterFactories)); + for (String analyzerName : oldAnalyzers.keySet()) { + NamedAnalyzer analyzer = rebuildIfNecessary(oldAnalyzers.get(analyzerName), indexSettings, currentCharFilterFactories, + currentTokenizerFactories, newTokenFilterFactories); + newAnalyzers.put(analyzerName, analyzer); } IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(currentTokenizerFactories, currentCharFilterFactories, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 70de937d6ed77..1afddeccd874f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -852,7 +852,7 @@ public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) th logger.info("reloading search analyzers"); // refresh indexAnalyzers and search analyzers - this.indexAnalyzers = registry.rebuildIndexAnalyzers(this.indexAnalyzers, indexSettings); + this.indexAnalyzers = registry.reloadIndexAnalyzers(this.indexAnalyzers, indexSettings); this.searchAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 845446de7786b..375f9b1233397 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -325,12 +325,6 @@ public void testRebuildIfNecessary() throws IOException { assertSame(noReloadingEither, AnalysisRegistry.rebuildIfNecessary(noReloadingEither, null, null, null, null)); - Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") - .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") - .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); - final AtomicInteger factoryCounter = new AtomicInteger(0); TestAnalysis testAnalysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisPlugin() { @@ -351,9 +345,14 @@ public TokenFilterFactory get(IndexSettings indexSettings, Environment environme tokenFilters[0] = testAnalysis.tokenFilter.get("myReloadableFilter"); NamedAnalyzer reloadableAnalyzer = new NamedAnalyzer("reloadableAnalyzer", AnalyzerScope.INDEX, new CustomAnalyzer("tokenizer", null, null, tokenFilters)); - IndexSettings indexSetings = new IndexSettings(IndexMetaData.builder("testIndex").settings(indexSettings).build(), indexSettings); + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") + .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); + IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("testIndex").settings(settings).build(), settings); - NamedAnalyzer rebuilt = AnalysisRegistry.rebuildIfNecessary(reloadableAnalyzer, indexSetings, testAnalysis.charFilter, + NamedAnalyzer rebuilt = AnalysisRegistry.rebuildIfNecessary(reloadableAnalyzer, indexSettings, testAnalysis.charFilter, testAnalysis.tokenizer, testAnalysis.tokenFilter); assertEquals(reloadableAnalyzer.name(), rebuilt.name()); assertNotSame(reloadableAnalyzer, rebuilt); @@ -363,12 +362,62 @@ public TokenFilterFactory get(IndexSettings indexSettings, Environment environme assertEquals(2, MyReloadableFilter.constructorCounter); } + public void testRebuildIndexAnalyzers() throws IOException { + + final AtomicInteger factoryCounter = new AtomicInteger(0); + AnalysisPlugin testPlugin = new AnalysisPlugin() { + + @Override + public Map> getTokenFilters() { + return Collections.singletonMap("myReloadableFilter", new AnalysisProvider() { + + @Override + public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) + throws IOException { + factoryCounter.getAndIncrement(); + return new MyReloadableFilter(); + } + }); + } + }; + + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") + .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); + AnalysisModule analysisModule = new AnalysisModule(TestEnvironment.newEnvironment(settings), singletonList(testPlugin)); + AnalysisRegistry registry = analysisModule.getAnalysisRegistry(); + IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("testIndex").settings(settings).build(), settings); + IndexAnalyzers oldIndexAnalyzers = registry.build(indexSettings); + assertEquals(1, factoryCounter.get()); + + IndexAnalyzers rebuildAnalyzers = registry.reloadIndexAnalyzers(oldIndexAnalyzers, indexSettings); + assertNotSame(oldIndexAnalyzers, rebuildAnalyzers); + assertEquals(2, factoryCounter.get()); + assertSame(oldIndexAnalyzers.getDefaultIndexAnalyzer(), rebuildAnalyzers.getDefaultIndexAnalyzer()); + assertSame(oldIndexAnalyzers.getDefaultSearchAnalyzer(), rebuildAnalyzers.getDefaultSearchAnalyzer()); + assertSame(oldIndexAnalyzers.getDefaultSearchQuoteAnalyzer(), rebuildAnalyzers.getDefaultSearchQuoteAnalyzer()); + assertNotSame(oldIndexAnalyzers.getAnalyzers(), rebuildAnalyzers.getAnalyzers()); + assertEquals(oldIndexAnalyzers.getAnalyzers().size(), rebuildAnalyzers.getAnalyzers().size()); + NamedAnalyzer oldVersion = oldIndexAnalyzers.get("reloadableAnalyzer"); + NamedAnalyzer newVersion = rebuildAnalyzers.get("reloadableAnalyzer"); + assertNotSame(oldVersion, newVersion); + assertThat(((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); + assertEquals(1, ((MyReloadableFilter) ((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0]).generation); + assertThat(((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); + assertEquals(2, ((MyReloadableFilter) ((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0]).generation); + } + static class MyReloadableFilter implements TokenFilterFactory { static int constructorCounter = 0; + private final int generation; MyReloadableFilter() { constructorCounter++; + generation = constructorCounter; } @Override @@ -378,11 +427,12 @@ public String name() { @Override public TokenStream create(TokenStream tokenStream) { - return null; + return tokenStream; } @Override public AnalysisMode getAnalysisMode() { return AnalysisMode.SEARCH_TIME; } - }; + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 6b36f985c210b..2f5de3f3d5b05 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -29,6 +29,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; + import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; From abb6523b33662345475851cbec31b8c0af5599e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 23 Apr 2019 19:13:12 +0200 Subject: [PATCH 08/20] More tests --- .../index/analysis/AnalysisRegistry.java | 7 +- .../index/analysis/IndexAnalyzers.java | 22 +++- .../index/mapper/MapperService.java | 9 +- .../index/analysis/AnalysisRegistryTests.java | 20 ++-- .../index/mapper/MapperServiceTests.java | 104 +++++++++++++++++- 5 files changed, 143 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 41f295571cc9b..4b72e6a58649f 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -539,7 +539,8 @@ private void processNormalizerFactory( * the same instance of {@link IndexAnalyzers} is returned. Otherwise, analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried * to be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. */ - public IndexAnalyzers reloadIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { + public IndexAnalyzers reloadIndexAnalyzers(IndexAnalyzers indexAnalyzers) throws IOException { + IndexSettings indexSettings = indexAnalyzers.getIndexSettings(); // scan analyzers to collect token filters that we need to reload Map oldAnalyzers = indexAnalyzers.getAnalyzers(); List analyzers = new ArrayList<>(oldAnalyzers.values()); @@ -578,9 +579,7 @@ public IndexAnalyzers reloadIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexS IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(currentTokenizerFactories, currentCharFilterFactories, newTokenFilterFactories); - return new IndexAnalyzers(indexSettings, indexAnalyzers.getDefaultIndexAnalyzer(), newDefaultSearchAnalyzer, - newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers(), - analysisProviders); + return new IndexAnalyzers(indexAnalyzers, newDefaultSearchAnalyzer, newDefaultSearchQuoteAnalyzer, newAnalyzers, analysisProviders); } static Set filtersThatNeedReloading(List analyzers) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index af0b8bb2c0d3d..e09ddd8c60c3a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -71,6 +71,22 @@ public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAna this.analysisProviders = analysisProviders; } + /** + * Partial copy-constructor that keeps references to settings, default index analyzer and normalizers from original + * {@link IndexAnalyzers} passed in but takes other search time analyzers as inputs. + */ + IndexAnalyzers(IndexAnalyzers original, NamedAnalyzer defaultSearchAnalyzer, NamedAnalyzer defaultSearchQuoteAnalyzer, + Map analyzers, IndexAnalysisProviders analysisProviders) { + super(original.getIndexSettings()); + this.defaultIndexAnalyzer = original.defaultIndexAnalyzer; + this.defaultSearchAnalyzer = defaultSearchAnalyzer; + this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer; + this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); + this.normalizers = original.normalizers; + this.whitespaceNormalizers = original.whitespaceNormalizers; + this.analysisProviders = analysisProviders; + } + /** * Returns an analyzer mapped to the given name or null if not present */ @@ -134,15 +150,15 @@ public NamedAnalyzer getDefaultSearchQuoteAnalyzer() { return defaultSearchQuoteAnalyzer; } - public Map getTokenizerFactoryFactories() { + Map getTokenizerFactoryFactories() { return analysisProviders.tokenizerFactoryFactories; } - public Map getCharFilterFactoryFactories() { + Map getCharFilterFactoryFactories() { return analysisProviders.charFilterFactoryFactories; } - public Map getTokenFilterFactoryFactories() { + Map getTokenFilterFactoryFactories() { return analysisProviders.tokenFilterFactoryFactories; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 1afddeccd874f..c4f93533ffb28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -850,14 +850,17 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { logger.info("reloading search analyzers"); - // refresh indexAnalyzers and search analyzers - this.indexAnalyzers = registry.reloadIndexAnalyzers(this.indexAnalyzers, indexSettings); + IndexAnalyzers reloadedIndexAnalyzers = registry.reloadIndexAnalyzers(this.indexAnalyzers); + if (indexAnalyzers == reloadedIndexAnalyzers) { + // nothing changed, so we can simply return; + return; + } + this.indexAnalyzers = reloadedIndexAnalyzers; this.searchAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); - // also reload search time analyzers in MappedFieldTypes // refresh search time analyzers in MappedFieldTypes List mftsToRefresh = StreamSupport.stream(fieldTypes.spliterator(), false) .filter(mft -> (mft.searchAnalyzer() != null && mft.searchAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME) diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 375f9b1233397..283a358de0287 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -352,6 +352,7 @@ public TokenFilterFactory get(IndexSettings indexSettings, Environment environme .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("testIndex").settings(settings).build(), settings); + int initialFilterCreationCount = MyReloadableFilter.constructorCounter.get(); NamedAnalyzer rebuilt = AnalysisRegistry.rebuildIfNecessary(reloadableAnalyzer, indexSettings, testAnalysis.charFilter, testAnalysis.tokenizer, testAnalysis.tokenFilter); assertEquals(reloadableAnalyzer.name(), rebuilt.name()); @@ -359,7 +360,8 @@ public TokenFilterFactory get(IndexSettings indexSettings, Environment environme assertEquals(2, factoryCounter.get()); // once on intialization, once again for reloading TokenFilterFactory reloadedFactory = ((CustomAnalyzer) rebuilt.analyzer()).tokenFilters()[0]; assertThat(reloadedFactory, instanceOf(MyReloadableFilter.class)); - assertEquals(2, MyReloadableFilter.constructorCounter); + // the filter factories should not be used at this poing since the function only re-creates the analyzer + assertEquals(initialFilterCreationCount, MyReloadableFilter.constructorCounter.get()); } public void testRebuildIndexAnalyzers() throws IOException { @@ -393,31 +395,33 @@ public TokenFilterFactory get(IndexSettings indexSettings, Environment environme IndexAnalyzers oldIndexAnalyzers = registry.build(indexSettings); assertEquals(1, factoryCounter.get()); - IndexAnalyzers rebuildAnalyzers = registry.reloadIndexAnalyzers(oldIndexAnalyzers, indexSettings); + IndexAnalyzers rebuildAnalyzers = registry.reloadIndexAnalyzers(oldIndexAnalyzers); assertNotSame(oldIndexAnalyzers, rebuildAnalyzers); assertEquals(2, factoryCounter.get()); assertSame(oldIndexAnalyzers.getDefaultIndexAnalyzer(), rebuildAnalyzers.getDefaultIndexAnalyzer()); assertSame(oldIndexAnalyzers.getDefaultSearchAnalyzer(), rebuildAnalyzers.getDefaultSearchAnalyzer()); assertSame(oldIndexAnalyzers.getDefaultSearchQuoteAnalyzer(), rebuildAnalyzers.getDefaultSearchQuoteAnalyzer()); + assertSame(oldIndexAnalyzers.getNormalizers(), rebuildAnalyzers.getNormalizers()); + assertSame(oldIndexAnalyzers.getWhitespaceNormalizers(), rebuildAnalyzers.getWhitespaceNormalizers()); assertNotSame(oldIndexAnalyzers.getAnalyzers(), rebuildAnalyzers.getAnalyzers()); assertEquals(oldIndexAnalyzers.getAnalyzers().size(), rebuildAnalyzers.getAnalyzers().size()); NamedAnalyzer oldVersion = oldIndexAnalyzers.get("reloadableAnalyzer"); NamedAnalyzer newVersion = rebuildAnalyzers.get("reloadableAnalyzer"); assertNotSame(oldVersion, newVersion); assertThat(((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); - assertEquals(1, ((MyReloadableFilter) ((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0]).generation); + int oldGeneration = ((MyReloadableFilter) ((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0]).generation.get(); assertThat(((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); - assertEquals(2, ((MyReloadableFilter) ((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0]).generation); + assertEquals(oldGeneration + 1, ((MyReloadableFilter) ((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0]).generation.get()); } static class MyReloadableFilter implements TokenFilterFactory { - static int constructorCounter = 0; - private final int generation; + private static AtomicInteger constructorCounter = new AtomicInteger(); + private final AtomicInteger generation; MyReloadableFilter() { - constructorCounter++; - generation = constructorCounter; + constructorCounter.getAndIncrement(); + generation = new AtomicInteger(constructorCounter.get()); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index d8c120e492d31..59dacb2679a37 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -27,11 +29,20 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisMode; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.indices.InvalidTypeNameException; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -39,6 +50,8 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.hamcrest.CoreMatchers.containsString; @@ -49,7 +62,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Collections.singleton(InternalSettingsPlugin.class); + return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class); } public void testTypeNameStartsWithIllegalDot() { @@ -434,4 +447,93 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName()); } + public void testReloadSearchAnalyzersNoReload() throws IOException { + MapperService mapperService = createIndex("no_reload_index", Settings.EMPTY).mapperService(); + IndexAnalyzers current = mapperService.getIndexAnalyzers(); + mapperService.reloadSearchAnalyzers(getInstanceFromNode(AnalysisRegistry.class)); + assertSame(current, mapperService.getIndexAnalyzers()); + } + + public void testReloadSearchAnalyzers() throws IOException { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") + .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); + + MapperService mapperService = createIndex("test_index", settings).mapperService(); + CompressedXContent mapping = new CompressedXContent(BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties") + .startObject("field") + .field("type", "text") + .field("analyzer", "simple") + .field("search_analyzer", "reloadableAnalyzer") + .field("search_quote_analyzer", "stop") + .endObject() + .startObject("otherField") + .field("type", "text") + .field("analyzer", "standard") + .field("search_analyzer", "simple") + .field("search_quote_analyzer", "reloadableAnalyzer") + .endObject() + .endObject() + .endObject().endObject())); + + mapperService.merge("_doc", mapping, MergeReason.MAPPING_UPDATE); + IndexAnalyzers current = mapperService.getIndexAnalyzers(); + MappedFieldType fieldType = mapperService.fullName("field"); + NamedAnalyzer originalFieldsearchAnalyzer = fieldType.searchAnalyzer(); + NamedAnalyzer originalFieldsearchQuoteAnalyzer = fieldType.searchQuoteAnalyzer(); + + MappedFieldType otherFieldType = mapperService.fullName("otherField"); + NamedAnalyzer originalOtherFieldsearchAnalyzer = otherFieldType.searchAnalyzer(); + NamedAnalyzer originalOtherFieldsearchQuoteAnalyzer = otherFieldType.searchQuoteAnalyzer(); + + mapperService.reloadSearchAnalyzers(getInstanceFromNode(AnalysisRegistry.class)); + IndexAnalyzers updatedAnalyzers = mapperService.getIndexAnalyzers(); + assertNotSame(current, updatedAnalyzers); + assertSame(current.getDefaultIndexAnalyzer(), updatedAnalyzers.getDefaultIndexAnalyzer()); + assertSame(current.getDefaultSearchAnalyzer(), updatedAnalyzers.getDefaultSearchAnalyzer()); + assertSame(current.getDefaultSearchQuoteAnalyzer(), updatedAnalyzers.getDefaultSearchQuoteAnalyzer()); + + assertNotSame(originalFieldsearchAnalyzer, mapperService.fullName("field").searchAnalyzer()); + assertSame(originalOtherFieldsearchAnalyzer, mapperService.fullName("otherField").searchAnalyzer()); + + assertSame(originalFieldsearchQuoteAnalyzer, mapperService.fullName("field").searchQuoteAnalyzer()); + assertNotSame(originalOtherFieldsearchQuoteAnalyzer, mapperService.fullName("otherField").searchQuoteAnalyzer()); + + } + + public static final class ReloadableFilterPlugin extends Plugin implements AnalysisPlugin { + + @Override + public Map> getTokenFilters() { + return Collections.singletonMap("myReloadableFilter", new AnalysisProvider() { + + @Override + public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) + throws IOException { + return new TokenFilterFactory() { + + @Override + public String name() { + return "myReloadableFilter"; + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return tokenStream; + } + + @Override + public AnalysisMode getAnalysisMode() { + return AnalysisMode.SEARCH_TIME; + } + }; + } + }); + } + } + } From 66c7bfa4fe07697709bcc6d6ee12524b2f53c15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 24 Apr 2019 19:52:27 +0200 Subject: [PATCH 09/20] Adding ReloadAnalyzerAction --- .../analysis/common/SynonymAnalyzerIT.java | 10 +- .../analysis/common/SynonymAnalyzerTests.java | 5 +- .../elasticsearch/action/ActionModule.java | 3 + .../refresh/TransportShardRefreshAction.java | 12 -- .../reloadanalyzer/ReloadAnalyzerAction.java | 37 ++++++ .../ReloadAnalyzerRequestBuilder.java | 35 +++++ .../ReloadAnalyzersRequest.java | 98 ++++++++++++++ .../ReloadAnalyzersResponse.java | 38 ++++++ .../TransportReloadAnalyzersAction.java | 124 ++++++++++++++++++ .../client/IndicesAdminClient.java | 18 +++ .../org/elasticsearch/client/Requests.java | 10 ++ .../client/support/AbstractClient.java | 19 +++ .../indices/RestReloadAnalyzersAction.java | 59 +++++++++ 13 files changed, 449 insertions(+), 19 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerAction.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerRequestBuilder.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java index 2c6d66720d390..a8aad6f0921f7 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java @@ -22,6 +22,7 @@ import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; @@ -82,8 +83,8 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException { } assertTrue(Files.exists(synonymsFile)); assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) + .put("index.number_of_shards", cluster().numDataNodes() * 2) + .put("index.number_of_replicas", 1) .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") .put("analysis.analyzer.my_synonym_analyzer.filter", "my_synonym_filter") .put("analysis.filter.my_synonym_filter.type", "synonym") @@ -108,8 +109,9 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException { new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { out.println("foo, baz, buzz"); } - // TODO don't use refresh here but something more specific - assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + ReloadAnalyzersResponse reloadResponse = client().admin().indices().prepareReloadAnalyzers("test").execute().actionGet(); + assertNoFailures(reloadResponse); + assertEquals(cluster().numDataNodes(), reloadResponse.getSuccessfulShards()); analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); assertEquals(3, analyzeResponse.getTokens().size()); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java index 9175999bb687b..e96708cfa67f9 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java @@ -67,7 +67,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException { } assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() - .put("index.number_of_shards", 1) + .put("index.number_of_shards", 5) .put("index.number_of_replicas", 0) .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") .putList("analysis.analyzer.my_synonym_analyzer.filter", "lowercase", "my_synonym_filter") @@ -93,8 +93,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException { new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { out.println("foo, baz, buzz"); } - // TODO don't use refresh here but something more specific - assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + assertNoFailures(client().admin().indices().prepareReloadAnalyzers("test").execute().actionGet()); analyzeResponse = client().admin().indices().prepareAnalyze("test", "Foo").setAnalyzer("my_synonym_analyzer").get(); assertEquals(3, analyzeResponse.getTokens().size()); diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 2cfe66372115f..337dd28e60843 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -125,6 +125,8 @@ import org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzerAction; +import org.elasticsearch.action.admin.indices.reloadanalyzer.TransportReloadAnalyzersAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; @@ -509,6 +511,7 @@ public void reg actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class); actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class); actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class); + actions.register(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class); //Indexed scripts actions.register(PutStoredScriptAction.INSTANCE, TransportPutStoredScriptAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 2a30bd11e6566..c0a52ac8c0d6a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -58,12 +58,6 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind ActionListener> listener) { ActionListener.completeWith(listener, () -> { primary.refresh("api"); - try { - primary.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); - } catch (Exception ex) { - logger.error(ex.getLocalizedMessage(), ex); - return new PrimaryResult(null, null, ex); - } logger.trace("{} refresh request executed on primary", primary.shardId()); return new PrimaryResult<>(shardRequest, new ReplicationResponse()); }); @@ -72,12 +66,6 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind @Override protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica) { replica.refresh("api"); - try { - replica.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); - } catch (Exception ex) { - logger.error(ex.getLocalizedMessage(), ex); - return new ReplicaResult(ex); - } logger.trace("{} refresh request executed on replica", replica.shardId()); return new ReplicaResult(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerAction.java new file mode 100644 index 0000000000000..b432f65630b99 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerAction.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import org.elasticsearch.action.Action; + +public class ReloadAnalyzerAction extends Action { + + public static final ReloadAnalyzerAction INSTANCE = new ReloadAnalyzerAction(); + public static final String NAME = "indices:admin/reload_analyzers"; + + private ReloadAnalyzerAction() { + super(NAME); + } + + @Override + public ReloadAnalyzersResponse newResponse() { + return new ReloadAnalyzersResponse(); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerRequestBuilder.java new file mode 100644 index 0000000000000..0680699af901a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzerRequestBuilder.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +/** + * Builder for reloading of analyzers + */ +public class ReloadAnalyzerRequestBuilder + extends BroadcastOperationRequestBuilder { + + public ReloadAnalyzerRequestBuilder(ElasticsearchClient client, ReloadAnalyzerAction action, String... indices) { + super(client, action, new ReloadAnalyzersRequest(indices)); + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java new file mode 100644 index 0000000000000..61363a65d45f4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.broadcast.BroadcastRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * Request for reloading index search analyzers + */ +public class ReloadAnalyzersRequest extends BroadcastRequest { + + public ReloadAnalyzersRequest() { + } + + /** + * Constructs a new request for reloading index search analyzers for one or more indices + */ + public ReloadAnalyzersRequest(String... indices) { + this.indices = indices; + } + + /** + * Constructs a new request for reloading index search analyzers for one or more indices + */ + public ReloadAnalyzersRequest(Settings settings, String... indices) { + this.indices = indices; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + return validationException; + } + + @Override + public String[] indices() { + return indices; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public String toString() { + return "indices : " + Arrays.toString(indices); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ReloadAnalyzersRequest that = (ReloadAnalyzersRequest) o; + return Objects.equals(indicesOptions(), that.indicesOptions()) + && Arrays.equals(indices, that.indices); + } + + @Override + public int hashCode() { + return Objects.hash(indicesOptions(), Arrays.hashCode(indices)); + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java new file mode 100644 index 0000000000000..bfba818472838 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; + +import java.util.List; + +/** + * The response object that will be returned when reloading analyzers + */ +public class ReloadAnalyzersResponse extends BroadcastResponse { + + public ReloadAnalyzersResponse() { + } + + public ReloadAnalyzersResponse(int totalShards, int successfulShards, int failedShards, + List shardFailures) { + super(totalShards, successfulShards, failedShards, shardFailures); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java new file mode 100644 index 0000000000000..ac83f9caf212d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.PlainShardsIterator; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardsIterator; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + + +/** + * Indices clear cache action. + */ +public class TransportReloadAnalyzersAction extends TransportBroadcastByNodeAction { + + private final IndicesService indicesService; + + @Inject + public TransportReloadAnalyzersAction(ClusterService clusterService, TransportService transportService, IndicesService indicesService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(ReloadAnalyzerAction.NAME, clusterService, transportService, actionFilters, indexNameExpressionResolver, + ReloadAnalyzersRequest::new, ThreadPool.Names.MANAGEMENT, false); + this.indicesService = indicesService; + } + + @Override + protected EmptyResult readShardResult(StreamInput in) throws IOException { + return EmptyResult.readEmptyResultFrom(in); + } + + @Override + protected ReloadAnalyzersResponse newResponse(ReloadAnalyzersRequest request, int totalShards, int successfulShards, int failedShards, + List responses, List shardFailures, ClusterState clusterState) { + return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures); + } + + @Override + protected ReloadAnalyzersRequest readRequestFrom(StreamInput in) throws IOException { + final ReloadAnalyzersRequest request = new ReloadAnalyzersRequest(); + request.readFrom(in); + return request; + } + + @Override + protected EmptyResult shardOperation(ReloadAnalyzersRequest request, ShardRouting shardRouting) throws IOException { + logger.info("reloading analyzers for index shard " + shardRouting); + IndexService indexService = indicesService.indexService(shardRouting.index()); + indexService.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); + return EmptyResult.INSTANCE; + } + + /** + * The reload request should go to only one shard per node the index lives on + */ + @Override + protected ShardsIterator shards(ClusterState clusterState, ReloadAnalyzersRequest request, String[] concreteIndices) { + RoutingTable routingTable = clusterState.routingTable(); + List shards = new ArrayList<>(); + for (String index : concreteIndices) { + Set nodesCovered = new HashSet<>(); + IndexRoutingTable indexRoutingTable = routingTable.index(index); + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + for (ShardRouting shardRouting : indexShardRoutingTable) { + if (nodesCovered.contains(shardRouting.currentNodeId()) == false) { + shards.add(shardRouting); + nodesCovered.add(shardRouting.currentNodeId()); + } + } + } + } + logger.info("Determined shards for reloading: " + shards); + return new PlainShardsIterator(shards); + } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, ReloadAnalyzersRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected ClusterBlockException checkRequestBlock(ClusterState state, ReloadAnalyzersRequest request, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices); + } +} diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index d5a73981f29f1..7f4265a18514d 100644 --- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -77,6 +77,9 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzerRequestBuilder; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersRequest; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; @@ -819,4 +822,19 @@ public interface IndicesAdminClient extends ElasticsearchClient { * Swaps the index pointed to by an alias given all provided conditions are satisfied */ void rolloverIndex(RolloverRequest request, ActionListener listener); + + /** + * Reloads analyzers of one or more indices. + */ + ActionFuture reloadAnalyzers(ReloadAnalyzersRequest request); + + /** + * Reloads analyzers of one or more indices. + */ + void reloadAnalyzers(ReloadAnalyzersRequest request, ActionListener listener); + + /** + * Reloads analyzers of one or more indices. + */ + ReloadAnalyzerRequestBuilder prepareReloadAnalyzers(String... indices); } diff --git a/server/src/main/java/org/elasticsearch/client/Requests.java b/server/src/main/java/org/elasticsearch/client/Requests.java index 19ad2fb397edc..8e955f496c040 100644 --- a/server/src/main/java/org/elasticsearch/client/Requests.java +++ b/server/src/main/java/org/elasticsearch/client/Requests.java @@ -52,6 +52,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersRequest; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; @@ -535,4 +536,13 @@ public static SnapshotsStatusRequest snapshotsStatusRequest(String repository) { return new SnapshotsStatusRequest(repository); } + /** + * A request to reload search Analyzers indices settings. + * + * @param indices The indices to update the settings for. Use {@code null} or {@code _all} to executed against all indices. + * @return The request + */ + public static ReloadAnalyzersRequest reloadAnalyzersRequest(String... indices) { + return new ReloadAnalyzersRequest(indices); + } } diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index e79f0567babe6..f0bdafe1b89fb 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -207,6 +207,10 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzerAction; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzerRequestBuilder; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersRequest; +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder; @@ -1520,6 +1524,11 @@ public RefreshRequestBuilder prepareRefresh(String... indices) { return new RefreshRequestBuilder(this, RefreshAction.INSTANCE).setIndices(indices); } + @Override + public ReloadAnalyzerRequestBuilder prepareReloadAnalyzers(String... indices) { + return new ReloadAnalyzerRequestBuilder(this, ReloadAnalyzerAction.INSTANCE).setIndices(indices); + } + @Override public ActionFuture stats(final IndicesStatsRequest request) { return execute(IndicesStatsAction.INSTANCE, request); @@ -1725,6 +1734,16 @@ public ActionFuture getSettings(GetSettingsRequest request) public void getSettings(GetSettingsRequest request, ActionListener listener) { execute(GetSettingsAction.INSTANCE, request, listener); } + + @Override + public ActionFuture reloadAnalyzers(ReloadAnalyzersRequest request) { + return execute(ReloadAnalyzerAction.INSTANCE, request); + } + + @Override + public void reloadAnalyzers(final ReloadAnalyzersRequest request, final ActionListener listener) { + execute(ReloadAnalyzerAction.INSTANCE, request, listener); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java new file mode 100644 index 0000000000000..2ca34acf3bd68 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.Set; + +import static org.elasticsearch.client.Requests.reloadAnalyzersRequest; +public class RestReloadAnalyzersAction extends BaseRestHandler { + + public RestReloadAnalyzersAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, "/{index}/_reload_search_analyzers", this); + } + + @Override + public String getName() { + return "reload_search_analyzers_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + ReloadAnalyzersRequest reloadAnalyuersRequest = reloadAnalyzersRequest(Strings.splitStringByCommaToArray(request.param("index"))); + reloadAnalyuersRequest.indicesOptions(IndicesOptions.fromRequest(request, reloadAnalyuersRequest.indicesOptions())); + return channel -> client.admin().indices().reloadAnalyzers(reloadAnalyuersRequest, new RestToXContentListener<>(channel)); + } + + @Override + protected Set responseParams() { + return Settings.FORMAT_PARAMS; + } +} From 0ea6def9f72a7f95179a2c6a2304ed79ff7a60d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 26 Apr 2019 18:09:14 +0200 Subject: [PATCH 10/20] iter --- .../ReloadAnalyzersRequest.java | 43 +----------- .../ReloadAnalyzersResponse.java | 66 +++++++++++++++++- .../TransportReloadAnalyzersAction.java | 62 ++++++++++++++--- .../indices/RestReloadAnalyzersAction.java | 9 +-- .../ReloadAnalyzersResponseTest.java | 67 +++++++++++++++++++ 5 files changed, 189 insertions(+), 58 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java index 61363a65d45f4..0db7b00d64f74 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersRequest.java @@ -19,13 +19,8 @@ package org.elasticsearch.action.admin.indices.reloadanalyzer; -import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; -import java.io.IOException; import java.util.Arrays; import java.util.Objects; @@ -34,47 +29,11 @@ */ public class ReloadAnalyzersRequest extends BroadcastRequest { - public ReloadAnalyzersRequest() { - } - /** * Constructs a new request for reloading index search analyzers for one or more indices */ public ReloadAnalyzersRequest(String... indices) { - this.indices = indices; - } - - /** - * Constructs a new request for reloading index search analyzers for one or more indices - */ - public ReloadAnalyzersRequest(Settings settings, String... indices) { - this.indices = indices; - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - return validationException; - } - - @Override - public String[] indices() { - return indices; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - - @Override - public String toString() { - return "indices : " + Arrays.toString(indices); + super(indices); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java index bfba818472838..950f82b3f14e6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponse.java @@ -20,19 +20,83 @@ import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; /** * The response object that will be returned when reloading analyzers */ public class ReloadAnalyzersResponse extends BroadcastResponse { + private final Map> reloadedIndicesNodes; + public ReloadAnalyzersResponse() { + reloadedIndicesNodes = Collections.emptyMap(); } public ReloadAnalyzersResponse(int totalShards, int successfulShards, int failedShards, - List shardFailures) { + List shardFailures, Map> reloadedIndicesNodes) { super(totalShards, successfulShards, failedShards, shardFailures); + this.reloadedIndicesNodes = reloadedIndicesNodes; + } + + /** + * Override in subclass to add custom fields following the common `_shards` field + */ + @Override + protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { + builder.startArray("reloaded_nodes"); + for (Entry> indexNodesReloaded : reloadedIndicesNodes.entrySet()) { + builder.startObject(); + builder.field("index", indexNodesReloaded.getKey()); + builder.field("reloaded_node_ids", indexNodesReloaded.getValue()); + builder.endObject(); + } + builder.endArray(); + } + + @SuppressWarnings({ "unchecked" }) + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("reload_analyzer", + true, arg -> { + BroadcastResponse response = (BroadcastResponse) arg[0]; + List>> results = (List>>) arg[1]; + Map> reloadedNodeIds = new HashMap<>(); + for (Tuple> result : results) { + reloadedNodeIds.put(result.v1(), result.v2()); + } + return new ReloadAnalyzersResponse(response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(), + Arrays.asList(response.getShardFailures()), reloadedNodeIds); + }); + + @SuppressWarnings({ "unchecked" }) + private static final ConstructingObjectParser>, Void> ENTRY_PARSER = new ConstructingObjectParser<>( + "reload_analyzer.entry", true, arg -> { + String index = (String) arg[0]; + List nodeIds = (List) arg[1]; + return new Tuple<>(index, nodeIds); + }); + + static { + declareBroadcastFields(PARSER); + PARSER.declareObjectArray(constructorArg(), ENTRY_PARSER, new ParseField("reloaded_nodes")); + ENTRY_PARSER.declareString(constructorArg(), new ParseField("index")); + ENTRY_PARSER.declareStringArray(constructorArg(), new ParseField("reloaded_node_ids")); + } + + public static ReloadAnalyzersResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java index ac83f9caf212d..6c352996d4761 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java @@ -19,6 +19,9 @@ package org.elasticsearch.action.admin.indices.reloadanalyzer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.reloadanalyzer.TransportReloadAnalyzersAction.ReloadResult; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; @@ -35,6 +38,8 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.index.IndexService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; @@ -42,17 +47,20 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** * Indices clear cache action. */ -public class TransportReloadAnalyzersAction extends TransportBroadcastByNodeAction { +public class TransportReloadAnalyzersAction + extends TransportBroadcastByNodeAction { + private static final Logger logger = LogManager.getLogger(TransportReloadAnalyzersAction.class); private final IndicesService indicesService; @Inject @@ -64,14 +72,27 @@ public TransportReloadAnalyzersAction(ClusterService clusterService, TransportSe } @Override - protected EmptyResult readShardResult(StreamInput in) throws IOException { - return EmptyResult.readEmptyResultFrom(in); + protected ReloadResult readShardResult(StreamInput in) throws IOException { + ReloadResult reloadResult = new ReloadResult(); + reloadResult.readFrom(in); + return reloadResult; } @Override protected ReloadAnalyzersResponse newResponse(ReloadAnalyzersRequest request, int totalShards, int successfulShards, int failedShards, - List responses, List shardFailures, ClusterState clusterState) { - return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures); + List responses, List shardFailures, ClusterState clusterState) { + Map> reloadedIndicesNodes = new HashMap>(); + for (ReloadResult result : responses) { + if (reloadedIndicesNodes.containsKey(result.index)) { + List nodes = reloadedIndicesNodes.get(result.index); + nodes.add(result.nodeId); + } else { + List nodes = new ArrayList<>(); + nodes.add(result.nodeId); + reloadedIndicesNodes.put(result.index, nodes); + } + } + return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures, reloadedIndicesNodes); } @Override @@ -82,11 +103,36 @@ protected ReloadAnalyzersRequest readRequestFrom(StreamInput in) throws IOExcept } @Override - protected EmptyResult shardOperation(ReloadAnalyzersRequest request, ShardRouting shardRouting) throws IOException { + protected ReloadResult shardOperation(ReloadAnalyzersRequest request, ShardRouting shardRouting) throws IOException { logger.info("reloading analyzers for index shard " + shardRouting); IndexService indexService = indicesService.indexService(shardRouting.index()); indexService.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); - return EmptyResult.INSTANCE; + return new ReloadResult(shardRouting.index().getName(), "sdfhsjkd"); + } + + public static final class ReloadResult implements Streamable { + String index; + String nodeId; + + private ReloadResult(String index, String nodeId) { + this.index = index; + this.nodeId = nodeId; + } + + private ReloadResult() { + } + + @Override + public void readFrom(StreamInput in) throws IOException { + this.index = in.readString(); + this.nodeId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(nodeId); + } } /** diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java index 2ca34acf3bd68..245800f0d49c1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestReloadAnalyzersAction.java @@ -30,14 +30,14 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; -import java.util.Set; import static org.elasticsearch.client.Requests.reloadAnalyzersRequest; public class RestReloadAnalyzersAction extends BaseRestHandler { public RestReloadAnalyzersAction(Settings settings, RestController controller) { super(settings); - controller.registerHandler(RestRequest.Method.PUT, "/{index}/_reload_search_analyzers", this); + controller.registerHandler(RestRequest.Method.GET, "/{index}/_reload_search_analyzers", this); + controller.registerHandler(RestRequest.Method.POST, "/{index}/_reload_search_analyzers", this); } @Override @@ -51,9 +51,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC reloadAnalyuersRequest.indicesOptions(IndicesOptions.fromRequest(request, reloadAnalyuersRequest.indicesOptions())); return channel -> client.admin().indices().reloadAnalyzers(reloadAnalyuersRequest, new RestToXContentListener<>(channel)); } - - @Override - protected Set responseParams() { - return Settings.FORMAT_PARAMS; - } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java b/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java new file mode 100644 index 0000000000000..340d7e817b50c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.reloadanalyzer; + +import com.carrotsearch.randomizedtesting.annotations.Repeat; + +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Repeat(iterations = 50) +public class ReloadAnalyzersResponseTest extends AbstractBroadcastResponseTestCase { + + @Override + protected ReloadAnalyzersResponse createTestInstance(int totalShards, int successfulShards, int failedShards, + List failures) { + Map> reloadedIndicesNodes = new HashMap<>(); + int randomIndices = randomIntBetween(0, 5); + for (int i = 0; i < randomIndices; i++) { + List randomNodeIds = Arrays.asList(generateRandomStringArray(5, 5, false, true)); + reloadedIndicesNodes.put(randomAlphaOfLengthBetween(5, 10), randomNodeIds); + } + return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, failures, reloadedIndicesNodes); + } + + @Override + protected ReloadAnalyzersResponse doParseInstance(XContentParser parser) throws IOException { + return ReloadAnalyzersResponse.fromXContent(parser); + } + + @Override + public void testToXContent() { + Map> reloadedIndicesNodes = Collections.singletonMap("index", Collections.singletonList("nodeId")); + ReloadAnalyzersResponse response = new ReloadAnalyzersResponse(10, 5, 5, null, reloadedIndicesNodes); + String output = Strings.toString(response); + assertEquals( + "{\"_shards\":{\"total\":10,\"successful\":5,\"failed\":5}," + + "\"reloaded_nodes\":[{\"index\":\"index\",\"reloaded_node_ids\":[\"nodeId\"]}]" + + "}", + output); + } +} From c9ebe8d2e3e6424791c1a33a0e4c2408ccae4dce Mon Sep 17 00:00:00 2001 From: jimczi Date: Tue, 21 May 2019 16:18:15 +0200 Subject: [PATCH 11/20] Make CustomAnalyzer reloadable --- .../index/analysis/CustomAnalyzer.java | 127 +++++++++++++----- .../analysis/CustomAnalyzerProvider.java | 71 +++++++--- .../index/analysis/IndexAnalyzers.java | 2 +- .../index/mapper/MapperService.java | 52 +++---- 4 files changed, 169 insertions(+), 83 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index a41ee33564400..ae2ca0dfa326b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -22,73 +22,126 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util.CloseableThreadLocal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import java.io.Reader; +import java.util.Map; public final class CustomAnalyzer extends Analyzer { + private CloseableThreadLocal storedComponents = new CloseableThreadLocal<>(); + private volatile AnalyzerComponents current; - private final String tokenizerName; - private final TokenizerFactory tokenizerFactory; - - private final CharFilterFactory[] charFilters; + private final AnalysisMode analysisMode; - private final TokenFilterFactory[] tokenFilters; + private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { + @Override + public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { + CustomAnalyzer custom = (CustomAnalyzer) analyzer; + AnalyzerComponents components = custom.getStoredComponents(); + if (components == null || custom.shouldReload(components)) { + custom.setStoredComponents(custom.current); + return null; + } + TokenStreamComponents tokenStream = (TokenStreamComponents) getStoredValue(analyzer); + assert tokenStream != null; + return tokenStream; + } - private final int positionIncrementGap; - private final int offsetGap; - private final AnalysisMode analysisMode; + @Override + public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents tokenStream) { + setStoredValue(analyzer, tokenStream); + } + }; public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, - TokenFilterFactory[] tokenFilters) { - this(tokenizerName, tokenizerFactory, charFilters, tokenFilters, 0, -1); + TokenFilterFactory[] tokenFilters) { + this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, -1, -1)); } public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, - TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) { - this.tokenizerName = tokenizerName; - this.tokenizerFactory = tokenizerFactory; - this.charFilters = charFilters; - this.tokenFilters = tokenFilters; - this.positionIncrementGap = positionIncrementGap; - this.offsetGap = offsetGap; + TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) { + this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, positionIncrementGap, offsetGap)); + } + + public CustomAnalyzer(AnalyzerComponents components) { + super(UPDATE_STRATEGY); + this.current = components; // merge and transfer token filter analysis modes with analyzer AnalysisMode mode = AnalysisMode.ALL; - for (TokenFilterFactory f : tokenFilters) { + for (TokenFilterFactory f : current.tokenFilters) { mode = mode.merge(f.getAnalysisMode()); } this.analysisMode = mode; } + /** + * TODO: We should not expose functions that return objects from the current, + * only the full {@link AnalyzerComponents} should be returned + */ + /** * The name of the tokenizer as configured by the user. */ public String getTokenizerName() { - return tokenizerName; + return current.tokenizerName; } public TokenizerFactory tokenizerFactory() { - return tokenizerFactory; + return current.tokenizerFactory; } public TokenFilterFactory[] tokenFilters() { - return tokenFilters; + return current.tokenFilters; } public CharFilterFactory[] charFilters() { - return charFilters; + return current.charFilters; } @Override public int getPositionIncrementGap(String fieldName) { - return this.positionIncrementGap; + return current.positionIncrementGap; + } + + private boolean shouldReload(AnalyzerComponents source) { + return this.current != source; + } + + public synchronized void reload(String name, + Settings settings, + final Map tokenizers, + final Map charFilters, + final Map tokenFilters) { + AnalyzerComponents components = CustomAnalyzerProvider.createComponents(name, settings, tokenizers, charFilters, tokenFilters); + this.current = components; + } + + @Override + public void close() { + storedComponents.close(); + } + + void setStoredComponents(AnalyzerComponents components) { + storedComponents.set(components); + } + + AnalyzerComponents getStoredComponents() { + return storedComponents.get(); + } + + public AnalyzerComponents getComponents() { + return current; } @Override public int getOffsetGap(String field) { - if (offsetGap < 0) { + final AnalyzerComponents components = getComponents(); + if (components.offsetGap < 0) { return super.getOffsetGap(field); } - return this.offsetGap; + return components.offsetGap; } public AnalysisMode getAnalysisMode() { @@ -97,9 +150,10 @@ public AnalysisMode getAnalysisMode() { @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = tokenizerFactory.create(); + final AnalyzerComponents components = getStoredComponents(); + Tokenizer tokenizer = components.tokenizerFactory.create(); TokenStream tokenStream = tokenizer; - for (TokenFilterFactory tokenFilter : tokenFilters) { + for (TokenFilterFactory tokenFilter : components.tokenFilters) { tokenStream = tokenFilter.create(tokenStream); } return new TokenStreamComponents(tokenizer, tokenStream); @@ -107,8 +161,9 @@ protected TokenStreamComponents createComponents(String fieldName) { @Override protected Reader initReader(String fieldName, Reader reader) { - if (charFilters != null && charFilters.length > 0) { - for (CharFilterFactory charFilter : charFilters) { + final AnalyzerComponents components = getStoredComponents(); + if (components.charFilters != null && components.charFilters.length > 0) { + for (CharFilterFactory charFilter : components.charFilters) { reader = charFilter.create(reader); } } @@ -117,7 +172,8 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected Reader initReaderForNormalization(String fieldName, Reader reader) { - for (CharFilterFactory charFilter : charFilters) { + final AnalyzerComponents components = getComponents(); + for (CharFilterFactory charFilter : components.charFilters) { reader = charFilter.normalize(reader); } return reader; @@ -125,10 +181,11 @@ protected Reader initReaderForNormalization(String fieldName, Reader reader) { @Override protected TokenStream normalize(String fieldName, TokenStream in) { - TokenStream result = in; - for (TokenFilterFactory filter : tokenFilters) { - result = filter.normalize(result); - } - return result; + final AnalyzerComponents components = getComponents(); + TokenStream result = in; + for (TokenFilterFactory filter : components.tokenFilters) { + result = filter.normalize(result); + } + return result; } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 8080a6af876a4..750c09ebb33b6 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -43,25 +43,35 @@ public CustomAnalyzerProvider(IndexSettings indexSettings, this.analyzerSettings = settings; } - void build(final Map tokenizers, final Map charFilters, - final Map tokenFilters) { - String tokenizerName = analyzerSettings.get("tokenizer"); + void build(final Map tokenizers, + final Map charFilters, + final Map tokenFilters) { + AnalyzerComponents components = createComponents(name(), analyzerSettings, tokenizers, charFilters, tokenFilters); + customAnalyzer = new CustomAnalyzer(components); + } + + public static AnalyzerComponents createComponents(String name, + Settings settings, + final Map tokenizers, + final Map charFilters, + final Map tokenFilters) { + String tokenizerName = settings.get("tokenizer"); if (tokenizerName == null) { - throw new IllegalArgumentException("Custom Analyzer [" + name() + "] must be configured with a tokenizer"); + throw new IllegalArgumentException("Custom Analyzer [" + name + "] must be configured with a tokenizer"); } TokenizerFactory tokenizer = tokenizers.get(tokenizerName); if (tokenizer == null) { - throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find tokenizer under name " + + throw new IllegalArgumentException("Custom Analyzer [" + name + "] failed to find tokenizer under name " + "[" + tokenizerName + "]"); } - List charFilterNames = analyzerSettings.getAsList("char_filter"); + List charFilterNames = settings.getAsList("char_filter"); List charFiltersList = new ArrayList<>(charFilterNames.size()); for (String charFilterName : charFilterNames) { CharFilterFactory charFilter = charFilters.get(charFilterName); if (charFilter == null) { - throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find char_filter under name " + + throw new IllegalArgumentException("Custom Analyzer [" + name + "] failed to find char_filter under name " + "[" + charFilterName + "]"); } charFiltersList.add(charFilter); @@ -69,27 +79,27 @@ void build(final Map tokenizers, final Map tokenFilterNames = analyzerSettings.getAsList("filter"); + List tokenFilterNames = settings.getAsList("filter"); List tokenFilterList = new ArrayList<>(tokenFilterNames.size()); for (String tokenFilterName : tokenFilterNames) { TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName); if (tokenFilter == null) { - throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name " + + throw new IllegalArgumentException("Custom Analyzer [" + name + "] failed to find filter under name " + "[" + tokenFilterName + "]"); } tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get); tokenFilterList.add(tokenFilter); } - this.customAnalyzer = new CustomAnalyzer(tokenizerName, tokenizer, - charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]), - tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]), - positionIncrementGap, - offsetGap + return new AnalyzerComponents(tokenizerName, tokenizer, + charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]), + tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]), + positionIncrementGap, + offsetGap ); } @@ -97,4 +107,33 @@ void build(final Map tokenizers, final Map getAnalyzers() { + public Map getAnalyzers() { return analyzers; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c4f93533ffb28..ddcbedf53b50a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; @@ -48,8 +49,12 @@ import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityService; @@ -72,9 +77,7 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -126,7 +129,7 @@ public enum MergeReason { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(MapperService.class)); - private volatile IndexAnalyzers indexAnalyzers; + private final IndexAnalyzers indexAnalyzers; private volatile String defaultMappingSource; @@ -140,8 +143,8 @@ public enum MergeReason { private final DocumentMapperParser documentParser; private final MapperAnalyzerWrapper indexAnalyzer; - private volatile MapperAnalyzerWrapper searchAnalyzer; - private volatile MapperAnalyzerWrapper searchQuoteAnalyzer; + private final MapperAnalyzerWrapper searchAnalyzer; + private final MapperAnalyzerWrapper searchQuoteAnalyzer; private volatile Map unmappedFieldTypes = emptyMap(); @@ -848,34 +851,21 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { } } - public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { + public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { logger.info("reloading search analyzers"); // refresh indexAnalyzers and search analyzers - IndexAnalyzers reloadedIndexAnalyzers = registry.reloadIndexAnalyzers(this.indexAnalyzers); - if (indexAnalyzers == reloadedIndexAnalyzers) { - // nothing changed, so we can simply return; - return; - } - this.indexAnalyzers = reloadedIndexAnalyzers; - this.searchAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); - this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchQuoteAnalyzer(), - p -> p.searchQuoteAnalyzer()); - - // refresh search time analyzers in MappedFieldTypes - List mftsToRefresh = StreamSupport.stream(fieldTypes.spliterator(), false) - .filter(mft -> (mft.searchAnalyzer() != null && mft.searchAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME) - || (mft.searchQuoteAnalyzer() != null && mft.searchQuoteAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME)) - .collect(Collectors.toList()); - List updated = mftsToRefresh.stream().map(mft -> { - MappedFieldType newMft = mft.clone(); - newMft.setSearchAnalyzer(indexAnalyzers.get(mft.searchAnalyzer().name())); - newMft.setSearchQuoteAnalyzer(indexAnalyzers.get(mft.searchQuoteAnalyzer().name())); - newMft.freeze(); - return newMft; - }).collect(Collectors.toList()); - fieldTypes = fieldTypes.copyAndAddAll(updated); - if (mapper != null) { - mapper.root().updateFieldType(fieldTypes.fullNameToFieldType); + final Map tokenizerFactories = registry.buildTokenizerFactories(indexSettings); + final Map charFilterFactories = registry.buildCharFilterFactories(indexSettings); + final Map tokenFilterFactories = registry.buildTokenFilterFactories(indexSettings); + final Map settings = indexSettings.getSettings().getGroups("index.analysis.analyzer"); + for (NamedAnalyzer namedAnalyzer : indexAnalyzers.getAnalyzers().values()) { + if (namedAnalyzer.analyzer() instanceof CustomAnalyzer) { + CustomAnalyzer analyzer = (CustomAnalyzer) namedAnalyzer.analyzer(); + if (analyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) { + Settings analyzerSettings = settings.get(namedAnalyzer.name()); + analyzer.reload(namedAnalyzer.name(), analyzerSettings, tokenizerFactories, charFilterFactories, tokenFilterFactories); + } + } } } } From 4aadbdd83e34693696a28793024b4b2075dbeeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 22 May 2019 13:38:34 +0200 Subject: [PATCH 12/20] Removing unused code after switch to reloading analyzer components --- .../index/analysis/AnalysisRegistry.java | 87 --------- .../index/analysis/CustomAnalyzer.java | 119 ++++-------- .../analysis/CustomAnalyzerProvider.java | 16 +- .../index/analysis/IndexAnalyzers.java | 12 -- .../analysis/ReloadableCustomAnalyzer.java | 106 +++++++++++ .../index/mapper/MapperService.java | 13 +- ...java => ReloadAnalyzersResponseTests.java} | 5 +- .../index/analysis/AnalysisRegistryTests.java | 175 ------------------ .../index/mapper/MapperServiceTests.java | 36 ++-- 9 files changed, 179 insertions(+), 390 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java rename server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/{ReloadAnalyzersResponseTest.java => ReloadAnalyzersResponseTests.java} (93%) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index f6b0e4e1c3b5a..bf834782d4e67 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -34,16 +34,10 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; @@ -530,85 +524,4 @@ private void processNormalizerFactory( } normalizers.put(name, normalizer); } - - /** - * Creates an new IndexAnalyzer instance based on the existing one passed in. If there are no analysis components that need reloading, - * the same instance of {@link IndexAnalyzers} is returned. Otherwise, analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried - * to be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. - */ - public IndexAnalyzers reloadIndexAnalyzers(IndexAnalyzers indexAnalyzers) throws IOException { - IndexSettings indexSettings = indexAnalyzers.getIndexSettings(); - // scan analyzers to collect token filters that we need to reload - Map oldAnalyzers = indexAnalyzers.getAnalyzers(); - List analyzers = new ArrayList<>(oldAnalyzers.values()); - analyzers.add(indexAnalyzers.getDefaultIndexAnalyzer()); - analyzers.add(indexAnalyzers.getDefaultSearchAnalyzer()); - Set filtersThatNeedReloading = filtersThatNeedReloading(analyzers); - if (filtersThatNeedReloading.size() == 0) { - return indexAnalyzers; - } - final Map tokenFiltersToReload = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER) - .entrySet().stream() - .filter(entry -> filtersThatNeedReloading.contains(entry.getKey())) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); - - final Map newTokenFilterFactories = buildMapping(Component.FILTER, indexSettings, - tokenFiltersToReload, this.tokenFilters, prebuiltAnalysis.preConfiguredTokenFilters); - - // fill the rest of the token filter factory map with the entries that are missing (were not reloaded) - for (Entry entry : indexAnalyzers.getTokenFilterFactoryFactories().entrySet()) { - newTokenFilterFactories.putIfAbsent(entry.getKey(), entry.getValue()); - } - - // char filters and tokenizers are not updateable, so we can use the old ones - Map currentCharFilterFactories = indexAnalyzers.getCharFilterFactoryFactories(); - Map currentTokenizerFactories = indexAnalyzers.getTokenizerFactoryFactories(); - Map newAnalyzers = new HashMap<>(); - for (String analyzerName : oldAnalyzers.keySet()) { - NamedAnalyzer analyzer = rebuildIfNecessary(oldAnalyzers.get(analyzerName), indexSettings, currentCharFilterFactories, - currentTokenizerFactories, newTokenFilterFactories); - newAnalyzers.put(analyzerName, analyzer); - } - - IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(currentTokenizerFactories, currentCharFilterFactories, - newTokenFilterFactories); - return new IndexAnalyzers(indexAnalyzers, newAnalyzers, analysisProviders); - } - - static Set filtersThatNeedReloading(List analyzers) { - Set filters = new HashSet<>(); - for (NamedAnalyzer namedAnalyzer : analyzers) { - // only rebuild custom analyzers that are in SEARCH_TIME mode - if ((namedAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == true - && (namedAnalyzer.analyzer() instanceof CustomAnalyzer == true)) { - TokenFilterFactory[] currentTokenFilters = ((CustomAnalyzer) namedAnalyzer.analyzer()).tokenFilters(); - // get token filters necessary to re-build the analyzer - Arrays.stream(currentTokenFilters).filter(f -> f.getAnalysisMode() == AnalysisMode.SEARCH_TIME).map(f -> f.name()) - .collect(Collectors.toCollection(() -> filters)); - } - } - return filters; - } - - /** - * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider - * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. - */ - static NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings, - Map charFilterFactories, Map tokenizerFactories, - Map tokenFilterFactories) throws IOException { - // only rebuild custom analyzers that are in SEARCH_TIME mode - if ((oldAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == false - || (oldAnalyzer.analyzer() instanceof CustomAnalyzer == false)) { - return oldAnalyzer; - } else { - String analyzerName = oldAnalyzer.name(); - Settings analyzerSettings = indexSettings.getSettings().getAsSettings("index.analysis.analyzer." + analyzerName); - - AnalyzerProvider analyzerProvider = new CustomAnalyzerProvider(indexSettings, analyzerName, analyzerSettings); - return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, tokenFilterFactories, charFilterFactories, - tokenizerFactories); - } - } - } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index ae2ca0dfa326b..89c4cc6e04dab 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -22,58 +22,24 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.util.CloseableThreadLocal; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import java.io.Reader; -import java.util.Map; -public final class CustomAnalyzer extends Analyzer { - private CloseableThreadLocal storedComponents = new CloseableThreadLocal<>(); - private volatile AnalyzerComponents current; +public class CustomAnalyzer extends Analyzer { private final AnalysisMode analysisMode; - - private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { - @Override - public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { - CustomAnalyzer custom = (CustomAnalyzer) analyzer; - AnalyzerComponents components = custom.getStoredComponents(); - if (components == null || custom.shouldReload(components)) { - custom.setStoredComponents(custom.current); - return null; - } - TokenStreamComponents tokenStream = (TokenStreamComponents) getStoredValue(analyzer); - assert tokenStream != null; - return tokenStream; - } - - @Override - public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents tokenStream) { - setStoredValue(analyzer, tokenStream); - } - }; + protected volatile AnalyzerComponents components; public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, TokenFilterFactory[] tokenFilters) { - this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, -1, -1)); + this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, 0, -1), GLOBAL_REUSE_STRATEGY); } - public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, - TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) { - this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, positionIncrementGap, offsetGap)); - } - - public CustomAnalyzer(AnalyzerComponents components) { - super(UPDATE_STRATEGY); - this.current = components; - // merge and transfer token filter analysis modes with analyzer - AnalysisMode mode = AnalysisMode.ALL; - for (TokenFilterFactory f : current.tokenFilters) { - mode = mode.merge(f.getAnalysisMode()); - } - this.analysisMode = mode; + CustomAnalyzer(AnalyzerComponents components, ReuseStrategy reuseStrategy) { + super(reuseStrategy); + this.components = components; + this.analysisMode = calculateAnalysisMode(components); } /** @@ -85,63 +51,36 @@ public CustomAnalyzer(AnalyzerComponents components) { * The name of the tokenizer as configured by the user. */ public String getTokenizerName() { - return current.tokenizerName; + return components.tokenizerName; } public TokenizerFactory tokenizerFactory() { - return current.tokenizerFactory; + return components.tokenizerFactory; } public TokenFilterFactory[] tokenFilters() { - return current.tokenFilters; + return components.tokenFilters; } public CharFilterFactory[] charFilters() { - return current.charFilters; + return components.charFilters; } @Override public int getPositionIncrementGap(String fieldName) { - return current.positionIncrementGap; + return components.positionIncrementGap; } - private boolean shouldReload(AnalyzerComponents source) { - return this.current != source; - } - - public synchronized void reload(String name, - Settings settings, - final Map tokenizers, - final Map charFilters, - final Map tokenFilters) { - AnalyzerComponents components = CustomAnalyzerProvider.createComponents(name, settings, tokenizers, charFilters, tokenFilters); - this.current = components; - } - - @Override - public void close() { - storedComponents.close(); - } - - void setStoredComponents(AnalyzerComponents components) { - storedComponents.set(components); - } - - AnalyzerComponents getStoredComponents() { - return storedComponents.get(); - } - - public AnalyzerComponents getComponents() { - return current; + protected AnalyzerComponents getComponents() { + return this.components; } @Override public int getOffsetGap(String field) { - final AnalyzerComponents components = getComponents(); - if (components.offsetGap < 0) { + if (this.components.offsetGap < 0) { return super.getOffsetGap(field); } - return components.offsetGap; + return this.components.offsetGap; } public AnalysisMode getAnalysisMode() { @@ -150,7 +89,7 @@ public AnalysisMode getAnalysisMode() { @Override protected TokenStreamComponents createComponents(String fieldName) { - final AnalyzerComponents components = getStoredComponents(); + final AnalyzerComponents components = getComponents(); Tokenizer tokenizer = components.tokenizerFactory.create(); TokenStream tokenStream = tokenizer; for (TokenFilterFactory tokenFilter : components.tokenFilters) { @@ -161,7 +100,7 @@ protected TokenStreamComponents createComponents(String fieldName) { @Override protected Reader initReader(String fieldName, Reader reader) { - final AnalyzerComponents components = getStoredComponents(); + final AnalyzerComponents components = getComponents(); if (components.charFilters != null && components.charFilters.length > 0) { for (CharFilterFactory charFilter : components.charFilters) { reader = charFilter.create(reader); @@ -188,4 +127,26 @@ protected TokenStream normalize(String fieldName, TokenStream in) { } return result; } + + private static AnalysisMode calculateAnalysisMode(AnalyzerComponents components) { + // merge and transfer token filter analysis modes with analyzer + AnalysisMode mode = AnalysisMode.ALL; + for (TokenFilterFactory f : components.tokenFilters) { + mode = mode.merge(f.getAnalysisMode()); + } + return mode; + } + + /** + * Factory method that either returns a plain {@link CustomAnalyzer} if the components used for creation are supporting index and search + * time use, or a {@link ReloadableCustomAnalyzer} if the components are intended for search time use only. + */ + static CustomAnalyzer create(AnalyzerComponents components) { + AnalysisMode mode = calculateAnalysisMode(components); + if (mode.equals(AnalysisMode.SEARCH_TIME)) { + return new ReloadableCustomAnalyzer(components); + } else { + return new CustomAnalyzer(components, GLOBAL_REUSE_STRATEGY); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 750c09ebb33b6..032ac16313168 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -47,11 +47,10 @@ void build(final Map tokenizers, final Map charFilters, final Map tokenFilters) { AnalyzerComponents components = createComponents(name(), analyzerSettings, tokenizers, charFilters, tokenFilters); - customAnalyzer = new CustomAnalyzer(components); + customAnalyzer = CustomAnalyzer.create(components); } - public static AnalyzerComponents createComponents(String name, - Settings settings, + static AnalyzerComponents createComponents(String name, Settings settings, final Map tokenizers, final Map charFilters, final Map tokenFilters) { @@ -108,7 +107,7 @@ public CustomAnalyzer get() { return this.customAnalyzer; } - public static class AnalyzerComponents { + static class AnalyzerComponents { final String tokenizerName; final TokenizerFactory tokenizerFactory; final CharFilterFactory[] charFilters; @@ -116,15 +115,6 @@ public static class AnalyzerComponents { final int positionIncrementGap; final int offsetGap; - public AnalyzerComponents(AnalyzerComponents clone) { - this.tokenizerName = clone.tokenizerName; - this.tokenizerFactory = clone.tokenizerFactory; - this.charFilters = clone.charFilters; - this.tokenFilters = clone.tokenFilters; - this.positionIncrementGap = clone.positionIncrementGap; - this.offsetGap = clone.offsetGap; - } - AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 667dee637820c..635714517162b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -67,18 +67,6 @@ public IndexAnalyzers(IndexSettings indexSettings, Map an this.analysisProviders = analysisProviders; } - /** - * Partial copy-constructor that keeps references to settings, default index analyzer and normalizers from original - * {@link IndexAnalyzers} passed in but takes other analyzers as inputs. - */ - IndexAnalyzers(IndexAnalyzers original, Map analyzers, IndexAnalysisProviders analysisProviders) { - super(original.getIndexSettings()); - this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); - this.normalizers = original.normalizers; - this.whitespaceNormalizers = original.whitespaceNormalizers; - this.analysisProviders = analysisProviders; - } - /** * Returns an analyzer mapped to the given name or null if not present */ diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java new file mode 100644 index 0000000000000..8506b2b5f427f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util.CloseableThreadLocal; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; + +import java.io.Reader; +import java.util.Map; + +public final class ReloadableCustomAnalyzer extends CustomAnalyzer { + private CloseableThreadLocal storedComponents = new CloseableThreadLocal<>(); + + private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { + @Override + public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { + ReloadableCustomAnalyzer custom = (ReloadableCustomAnalyzer) analyzer; + AnalyzerComponents components = custom.getStoredComponents(); + if (components == null || custom.shouldReload(components)) { + custom.setStoredComponents(custom.getComponents()); + return null; + } + TokenStreamComponents tokenStream = (TokenStreamComponents) getStoredValue(analyzer); + assert tokenStream != null; + return tokenStream; + } + + @Override + public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents tokenStream) { + setStoredValue(analyzer, tokenStream); + } + }; + + ReloadableCustomAnalyzer(AnalyzerComponents components) { + super(components, UPDATE_STRATEGY); + } + + private boolean shouldReload(AnalyzerComponents source) { + return this.components != source; + } + + public synchronized void reload(String name, + Settings settings, + final Map tokenizers, + final Map charFilters, + final Map tokenFilters) { + AnalyzerComponents components = CustomAnalyzerProvider.createComponents(name, settings, tokenizers, charFilters, tokenFilters); + this.components = components; + } + + @Override + public void close() { + storedComponents.close(); + } + + private void setStoredComponents(AnalyzerComponents components) { + storedComponents.set(components); + } + + private AnalyzerComponents getStoredComponents() { + return storedComponents.get(); + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final AnalyzerComponents components = getStoredComponents(); + Tokenizer tokenizer = components.tokenizerFactory.create(); + TokenStream tokenStream = tokenizer; + for (TokenFilterFactory tokenFilter : components.tokenFilters) { + tokenStream = tokenFilter.create(tokenStream); + } + return new TokenStreamComponents(tokenizer, tokenStream); + } + + @Override + protected Reader initReader(String fieldName, Reader reader) { + final AnalyzerComponents components = getStoredComponents(); + if (components.charFilters != null && components.charFilters.length > 0) { + for (CharFilterFactory charFilter : components.charFilters) { + reader = charFilter.create(reader); + } + } + return reader; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index ddcbedf53b50a..fc7c94372f16c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -47,12 +47,11 @@ import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; -import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.CharFilterFactory; -import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.mapper.Mapper.BuilderContext; @@ -859,12 +858,10 @@ public synchronized void reloadSearchAnalyzers(AnalysisRegistry registry) throws final Map tokenFilterFactories = registry.buildTokenFilterFactories(indexSettings); final Map settings = indexSettings.getSettings().getGroups("index.analysis.analyzer"); for (NamedAnalyzer namedAnalyzer : indexAnalyzers.getAnalyzers().values()) { - if (namedAnalyzer.analyzer() instanceof CustomAnalyzer) { - CustomAnalyzer analyzer = (CustomAnalyzer) namedAnalyzer.analyzer(); - if (analyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) { - Settings analyzerSettings = settings.get(namedAnalyzer.name()); - analyzer.reload(namedAnalyzer.name(), analyzerSettings, tokenizerFactories, charFilterFactories, tokenFilterFactories); - } + if (namedAnalyzer.analyzer() instanceof ReloadableCustomAnalyzer) { + ReloadableCustomAnalyzer analyzer = (ReloadableCustomAnalyzer) namedAnalyzer.analyzer(); + Settings analyzerSettings = settings.get(namedAnalyzer.name()); + analyzer.reload(namedAnalyzer.name(), analyzerSettings, tokenizerFactories, charFilterFactories, tokenFilterFactories); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java b/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java rename to server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTests.java index 340d7e817b50c..126a08d21e7b0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTest.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/reloadanalyzer/ReloadAnalyzersResponseTests.java @@ -19,8 +19,6 @@ package org.elasticsearch.action.admin.indices.reloadanalyzer; -import com.carrotsearch.randomizedtesting.annotations.Repeat; - import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase; import org.elasticsearch.common.Strings; @@ -33,8 +31,7 @@ import java.util.List; import java.util.Map; -@Repeat(iterations = 50) -public class ReloadAnalyzersResponseTest extends AbstractBroadcastResponseTestCase { +public class ReloadAnalyzersResponseTests extends AbstractBroadcastResponseTestCase { @Override protected ReloadAnalyzersResponse createTestInstance(int totalShards, int successfulShards, int failedShards, diff --git a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 0979bca883f72..f7437ad8ec5a9 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -44,12 +43,8 @@ import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -269,174 +264,4 @@ public void testEnsureCloseInvocationProperlyDelegated() throws IOException { registry.close(); verify(mock).close(); } - - /** - * test helper method that filters list of input analyzers to get the names of the token filters they contain - * that can be reloaded at search time - */ - public void testFiltersThatNeedReloading() { - TokenFilterFactory[] tokenFilters = new TokenFilterFactory[] { - createTokenFilter("first", AnalysisMode.ALL), - createTokenFilter("second", AnalysisMode.SEARCH_TIME), - createTokenFilter("third", AnalysisMode.ALL) - }; - List analyzers = Arrays.asList( - new NamedAnalyzer("myAnalyzer", AnalyzerScope.INDEX, new CustomAnalyzer("tokenizer", null, null, tokenFilters )), - new NamedAnalyzer("myAnalyzer", AnalyzerScope.INDEX, new StandardAnalyzer())); - Set filtersThatNeedReloading = AnalysisRegistry.filtersThatNeedReloading(analyzers); - assertEquals(1, filtersThatNeedReloading.size()); - assertTrue(filtersThatNeedReloading.contains("second")); - } - - private TokenFilterFactory createTokenFilter(String name, AnalysisMode mode) { - return new TokenFilterFactory() { - - @Override - public String name() { - return name; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return null; - } - - @Override - public AnalysisMode getAnalysisMode() { - return mode; - } - }; - } - - /** - * test helper function that rebuilds an input {@link NamedAnalyzer} if it is reloadable - */ - public void testRebuildIfNecessary() throws IOException { - NamedAnalyzer noReloading = new NamedAnalyzer("noReloading", AnalyzerScope.INDEX, new StandardAnalyzer()); - assertSame(noReloading, AnalysisRegistry.rebuildIfNecessary(noReloading, null, null, null, null)); - - TokenFilterFactory[] tokenFilters = new TokenFilterFactory[] { - createTokenFilter("first", AnalysisMode.INDEX_TIME), - createTokenFilter("second", AnalysisMode.ALL), - createTokenFilter("third", AnalysisMode.ALL) - }; - NamedAnalyzer noReloadingEither = new NamedAnalyzer("noReloadingEither", AnalyzerScope.INDEX, - new CustomAnalyzer("tokenizer", null, null, tokenFilters)); - assertSame(noReloadingEither, AnalysisRegistry.rebuildIfNecessary(noReloadingEither, null, null, null, null)); - - - final AtomicInteger factoryCounter = new AtomicInteger(0); - TestAnalysis testAnalysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisPlugin() { - - @Override - public Map> getTokenFilters() { - return Collections.singletonMap("myReloadableFilter", new AnalysisProvider() { - - @Override - public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) - throws IOException { - factoryCounter.getAndIncrement(); - return new MyReloadableFilter(); - } - }); - } - }); - - tokenFilters[0] = testAnalysis.tokenFilter.get("myReloadableFilter"); - NamedAnalyzer reloadableAnalyzer = new NamedAnalyzer("reloadableAnalyzer", AnalyzerScope.INDEX, - new CustomAnalyzer("tokenizer", null, null, tokenFilters)); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") - .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") - .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); - IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("testIndex").settings(settings).build(), settings); - - int initialFilterCreationCount = MyReloadableFilter.constructorCounter.get(); - NamedAnalyzer rebuilt = AnalysisRegistry.rebuildIfNecessary(reloadableAnalyzer, indexSettings, testAnalysis.charFilter, - testAnalysis.tokenizer, testAnalysis.tokenFilter); - assertEquals(reloadableAnalyzer.name(), rebuilt.name()); - assertNotSame(reloadableAnalyzer, rebuilt); - assertEquals(2, factoryCounter.get()); // once on intialization, once again for reloading - TokenFilterFactory reloadedFactory = ((CustomAnalyzer) rebuilt.analyzer()).tokenFilters()[0]; - assertThat(reloadedFactory, instanceOf(MyReloadableFilter.class)); - // the filter factories should not be used at this poing since the function only re-creates the analyzer - assertEquals(initialFilterCreationCount, MyReloadableFilter.constructorCounter.get()); - } - - public void testRebuildIndexAnalyzers() throws IOException { - - final AtomicInteger factoryCounter = new AtomicInteger(0); - AnalysisPlugin testPlugin = new AnalysisPlugin() { - - @Override - public Map> getTokenFilters() { - return Collections.singletonMap("myReloadableFilter", new AnalysisProvider() { - - @Override - public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) - throws IOException { - factoryCounter.getAndIncrement(); - return new MyReloadableFilter(); - } - }); - } - }; - - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom") - .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard") - .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build(); - AnalysisModule analysisModule = new AnalysisModule(TestEnvironment.newEnvironment(settings), singletonList(testPlugin)); - AnalysisRegistry registry = analysisModule.getAnalysisRegistry(); - IndexSettings indexSettings = new IndexSettings(IndexMetaData.builder("testIndex").settings(settings).build(), settings); - IndexAnalyzers oldIndexAnalyzers = registry.build(indexSettings); - assertEquals(1, factoryCounter.get()); - - IndexAnalyzers rebuildAnalyzers = registry.reloadIndexAnalyzers(oldIndexAnalyzers); - assertNotSame(oldIndexAnalyzers, rebuildAnalyzers); - assertEquals(2, factoryCounter.get()); - assertSame(oldIndexAnalyzers.getDefaultIndexAnalyzer(), rebuildAnalyzers.getDefaultIndexAnalyzer()); - assertSame(oldIndexAnalyzers.getDefaultSearchAnalyzer(), rebuildAnalyzers.getDefaultSearchAnalyzer()); - assertSame(oldIndexAnalyzers.getDefaultSearchQuoteAnalyzer(), rebuildAnalyzers.getDefaultSearchQuoteAnalyzer()); - assertSame(oldIndexAnalyzers.getNormalizers(), rebuildAnalyzers.getNormalizers()); - assertSame(oldIndexAnalyzers.getWhitespaceNormalizers(), rebuildAnalyzers.getWhitespaceNormalizers()); - assertNotSame(oldIndexAnalyzers.getAnalyzers(), rebuildAnalyzers.getAnalyzers()); - assertEquals(oldIndexAnalyzers.getAnalyzers().size(), rebuildAnalyzers.getAnalyzers().size()); - NamedAnalyzer oldVersion = oldIndexAnalyzers.get("reloadableAnalyzer"); - NamedAnalyzer newVersion = rebuildAnalyzers.get("reloadableAnalyzer"); - assertNotSame(oldVersion, newVersion); - assertThat(((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); - int oldGeneration = ((MyReloadableFilter) ((CustomAnalyzer) oldVersion.analyzer()).tokenFilters()[0]).generation.get(); - assertThat(((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0], instanceOf(MyReloadableFilter.class)); - assertEquals(oldGeneration + 1, ((MyReloadableFilter) ((CustomAnalyzer) newVersion.analyzer()).tokenFilters()[0]).generation.get()); - } - - static class MyReloadableFilter implements TokenFilterFactory { - - private static AtomicInteger constructorCounter = new AtomicInteger(); - private final AtomicInteger generation; - - MyReloadableFilter() { - constructorCounter.getAndIncrement(); - generation = new AtomicInteger(constructorCounter.get()); - } - - @Override - public String name() { - return "myReloadableFilter"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return tokenStream; - } - @Override - public AnalysisMode getAnalysisMode() { - return AnalysisMode.SEARCH_TIME; - } - } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 59dacb2679a37..d4c2f40a8edac 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -482,27 +483,38 @@ public void testReloadSearchAnalyzers() throws IOException { mapperService.merge("_doc", mapping, MergeReason.MAPPING_UPDATE); IndexAnalyzers current = mapperService.getIndexAnalyzers(); - MappedFieldType fieldType = mapperService.fullName("field"); - NamedAnalyzer originalFieldsearchAnalyzer = fieldType.searchAnalyzer(); - NamedAnalyzer originalFieldsearchQuoteAnalyzer = fieldType.searchQuoteAnalyzer(); - MappedFieldType otherFieldType = mapperService.fullName("otherField"); - NamedAnalyzer originalOtherFieldsearchAnalyzer = otherFieldType.searchAnalyzer(); - NamedAnalyzer originalOtherFieldsearchQuoteAnalyzer = otherFieldType.searchQuoteAnalyzer(); + ReloadableCustomAnalyzer originalReloadableAnalyzer = (ReloadableCustomAnalyzer) current.get("reloadableAnalyzer").analyzer(); + TokenFilterFactory[] originalTokenFilters = originalReloadableAnalyzer.tokenFilters(); + assertEquals(1, originalTokenFilters.length); + assertEquals("myReloadableFilter", originalTokenFilters[0].name()); + // now reload, this should change the tokenfilterFactory inside the analyzer mapperService.reloadSearchAnalyzers(getInstanceFromNode(AnalysisRegistry.class)); IndexAnalyzers updatedAnalyzers = mapperService.getIndexAnalyzers(); - assertNotSame(current, updatedAnalyzers); + assertSame(current, updatedAnalyzers); assertSame(current.getDefaultIndexAnalyzer(), updatedAnalyzers.getDefaultIndexAnalyzer()); assertSame(current.getDefaultSearchAnalyzer(), updatedAnalyzers.getDefaultSearchAnalyzer()); assertSame(current.getDefaultSearchQuoteAnalyzer(), updatedAnalyzers.getDefaultSearchQuoteAnalyzer()); - assertNotSame(originalFieldsearchAnalyzer, mapperService.fullName("field").searchAnalyzer()); - assertSame(originalOtherFieldsearchAnalyzer, mapperService.fullName("otherField").searchAnalyzer()); - - assertSame(originalFieldsearchQuoteAnalyzer, mapperService.fullName("field").searchQuoteAnalyzer()); - assertNotSame(originalOtherFieldsearchQuoteAnalyzer, mapperService.fullName("otherField").searchQuoteAnalyzer()); + assertFalse(assertSameContainedFilters(originalTokenFilters, current.get("reloadableAnalyzer"))); + assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fullName("field").searchAnalyzer())); + assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fullName("otherField").searchQuoteAnalyzer())); + } + private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { + ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); + TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.tokenFilters(); + assertEquals(originalTokenFilter.length, newTokenFilters.length); + int i = 0; + for (TokenFilterFactory tf : newTokenFilters ) { + assertEquals(originalTokenFilter[i].name(), tf.name()); + if (originalTokenFilter[i] != tf) { + return false; + } + i++; + } + return true; } public static final class ReloadableFilterPlugin extends Plugin implements AnalysisPlugin { From d486308b0cf80659cafa3e422f3ed8cdc0844e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 22 May 2019 15:39:26 +0200 Subject: [PATCH 13/20] Removing more stuff from IndexAnalyzers --- .../index/analysis/AnalysisRegistry.java | 5 +-- .../index/analysis/IndexAnalyzers.java | 44 ------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index bf834782d4e67..bd84253c1ac3a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -26,7 +26,6 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.IndexAnalyzers.IndexAnalysisProviders; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; @@ -453,9 +452,7 @@ public IndexAnalyzers build(IndexSettings indexSettings, throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\""); } } - IndexAnalysisProviders analysisProviders = new IndexAnalysisProviders(tokenizerFactoryFactories, charFilterFactoryFactories, - tokenFilterFactoryFactories); - return new IndexAnalyzers(indexSettings, analyzers, normalizers, whitespaceNormalizers, analysisProviders); + return new IndexAnalyzers(indexSettings, analyzers, normalizers, whitespaceNormalizers); } private static NamedAnalyzer produceAnalyzer(String name, diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 635714517162b..3ac22bc7b3df0 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -24,7 +24,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -46,15 +45,9 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos private final Map analyzers; private final Map normalizers; private final Map whitespaceNormalizers; - private final IndexAnalysisProviders analysisProviders; public IndexAnalyzers(IndexSettings indexSettings, Map analyzers, Map normalizers, Map whitespaceNormalizers) { - this(indexSettings, analyzers, normalizers, whitespaceNormalizers, IndexAnalysisProviders.EMPTY); - } - - public IndexAnalyzers(IndexSettings indexSettings, Map analyzers, Map normalizers, - Map whitespaceNormalizers, IndexAnalysisProviders analysisProviders) { super(indexSettings); Objects.requireNonNull(analyzers.get(DEFAULT_ANALYZER_NAME), "the default analyzer must be set"); if (analyzers.get(DEFAULT_ANALYZER_NAME).name().equals(DEFAULT_ANALYZER_NAME) == false) { @@ -64,7 +57,6 @@ public IndexAnalyzers(IndexSettings indexSettings, Map an this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); this.normalizers = unmodifiableMap(new HashMap<>(normalizers)); this.whitespaceNormalizers = unmodifiableMap(new HashMap<>(whitespaceNormalizers)); - this.analysisProviders = analysisProviders; } /** @@ -102,13 +94,6 @@ public NamedAnalyzer getWhitespaceNormalizer(String name) { return whitespaceNormalizers.get(name); } - /** - * Returns an (unmodifiable) map of containing the index whitespace normalizers - */ - Map getWhitespaceNormalizers() { - return whitespaceNormalizers; - } - /** * Returns the default index analyzer for this index */ @@ -130,39 +115,10 @@ public NamedAnalyzer getDefaultSearchQuoteAnalyzer() { return analyzers.getOrDefault(DEFAULT_SEARCH_QUOTED_ANALYZER_NAME, getDefaultSearchAnalyzer()); } - Map getTokenizerFactoryFactories() { - return analysisProviders.tokenizerFactoryFactories; - } - - Map getCharFilterFactoryFactories() { - return analysisProviders.charFilterFactoryFactories; - } - - Map getTokenFilterFactoryFactories() { - return analysisProviders.tokenFilterFactoryFactories; - } - @Override public void close() throws IOException { IOUtils.close(() -> Stream.concat(analyzers.values().stream(), normalizers.values().stream()) .filter(a -> a.scope() == AnalyzerScope.INDEX) .iterator()); } - - static class IndexAnalysisProviders { - - static final IndexAnalysisProviders EMPTY = new IndexAnalysisProviders(Collections.emptyMap(), Collections.emptyMap(), - Collections.emptyMap()); - - private final Map tokenizerFactoryFactories; - private final Map charFilterFactoryFactories; - private final Map tokenFilterFactoryFactories; - - IndexAnalysisProviders(Map tokenizerFactoryFactories, - Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { - this.tokenizerFactoryFactories = unmodifiableMap(tokenizerFactoryFactories); - this.charFilterFactoryFactories = unmodifiableMap(charFilterFactoryFactories); - this.tokenFilterFactoryFactories = unmodifiableMap(tokenFilterFactoryFactories); - } - } } From 66091a2cc8e9351ad1d1ee4e3aa015976cf1d55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 22 May 2019 15:47:54 +0200 Subject: [PATCH 14/20] Some more cleanups --- .../index/analysis/AnalysisRegistry.java | 2 +- .../index/analysis/IndexAnalyzers.java | 7 ------- .../elasticsearch/index/mapper/FieldTypeLookup.java | 13 ------------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index bd84253c1ac3a..917f1eab26b4c 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -276,7 +276,7 @@ public String toString() { } @SuppressWarnings("unchecked") - Map buildMapping(Component component, IndexSettings settings, Map settingsMap, + private Map buildMapping(Component component, IndexSettings settings, Map settingsMap, Map> providerMap, Map> defaultInstance) throws IOException { Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build(); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 3ac22bc7b3df0..ec1bfe62c5f7a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -80,13 +80,6 @@ public NamedAnalyzer getNormalizer(String name) { return normalizers.get(name); } - /** - * Returns an (unmodifiable) map of containing the index normalizers - */ - Map getNormalizers() { - return normalizers; - } - /** * Returns a normalizer that splits on whitespace mapped to the given name or null if not present */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 75ebc9eecc466..0dc8b6a00c09e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -86,19 +86,6 @@ public FieldTypeLookup copyAndAddAll(String type, return new FieldTypeLookup(fullName, aliases); } - /** - * Return a new instance that contains the union of this instance and the mapped field types - */ - public FieldTypeLookup copyAndAddAll(Collection mappedFieldTypes) { - CopyOnWriteHashMap fullName = this.fullNameToFieldType; - CopyOnWriteHashMap aliases = this.aliasToConcreteName; - - for (MappedFieldType mft : mappedFieldTypes) { - fullName = fullName.copyAndPut(mft.name(), mft); - } - return new FieldTypeLookup(fullName, aliases); - } - /** Returns the field for the given field */ public MappedFieldType get(String field) { String concreteField = aliasToConcreteName.getOrDefault(field, field); From b4fb376981c5b2dd8bdcd8090abfda071d4eb396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 22 May 2019 18:35:50 +0200 Subject: [PATCH 15/20] Move getters into AnalyzerComponents --- .../analyze/TransportAnalyzeAction.java | 10 ++-- .../index/analysis/CustomAnalyzer.java | 49 ++++--------------- .../analysis/CustomAnalyzerProvider.java | 38 +++++++++++--- .../index/analysis/NamedAnalyzer.java | 4 +- .../analysis/ReloadableCustomAnalyzer.java | 8 +-- .../highlight/FragmentBuilderHelper.java | 2 +- .../phrase/PhraseSuggestionBuilder.java | 2 +- .../index/mapper/MapperServiceTests.java | 4 +- .../indices/analysis/AnalysisModuleTests.java | 8 +-- 9 files changed, 62 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 55bd593742667..9333c424da52c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.CustomAnalyzer; +import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.NormalizingCharFilterFactory; @@ -308,9 +309,10 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy if (customAnalyzer != null) { // customAnalyzer = divide charfilter, tokenizer tokenfilters - CharFilterFactory[] charFilterFactories = customAnalyzer.charFilters(); - TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory(); - TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters(); + AnalyzerComponents components = customAnalyzer.getComponents(); + CharFilterFactory[] charFilterFactories = components.getCharFilters(); + TokenizerFactory tokenizerFactory = components.getTokenizerFactory(); + TokenFilterFactory[] tokenFilterFactories = components.getTokenFilters(); String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length]; TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? @@ -370,7 +372,7 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy } } detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList( - customAnalyzer.getTokenizerName(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); + components.getTokenizerName(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); } else { String name; if (analyzer instanceof NamedAnalyzer) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index 89c4cc6e04dab..ede17dfb08a22 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -42,45 +42,16 @@ public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, C this.analysisMode = calculateAnalysisMode(components); } - /** - * TODO: We should not expose functions that return objects from the current, - * only the full {@link AnalyzerComponents} should be returned - */ - - /** - * The name of the tokenizer as configured by the user. - */ - public String getTokenizerName() { - return components.tokenizerName; - } - - public TokenizerFactory tokenizerFactory() { - return components.tokenizerFactory; - } - - public TokenFilterFactory[] tokenFilters() { - return components.tokenFilters; - } - - public CharFilterFactory[] charFilters() { - return components.charFilters; - } - - @Override - public int getPositionIncrementGap(String fieldName) { - return components.positionIncrementGap; - } - - protected AnalyzerComponents getComponents() { + public AnalyzerComponents getComponents() { return this.components; } @Override public int getOffsetGap(String field) { - if (this.components.offsetGap < 0) { + if (this.components.getOffsetGap() < 0) { return super.getOffsetGap(field); } - return this.components.offsetGap; + return this.components.getOffsetGap(); } public AnalysisMode getAnalysisMode() { @@ -90,9 +61,9 @@ public AnalysisMode getAnalysisMode() { @Override protected TokenStreamComponents createComponents(String fieldName) { final AnalyzerComponents components = getComponents(); - Tokenizer tokenizer = components.tokenizerFactory.create(); + Tokenizer tokenizer = components.getTokenizerFactory().create(); TokenStream tokenStream = tokenizer; - for (TokenFilterFactory tokenFilter : components.tokenFilters) { + for (TokenFilterFactory tokenFilter : components.getTokenFilters()) { tokenStream = tokenFilter.create(tokenStream); } return new TokenStreamComponents(tokenizer, tokenStream); @@ -101,8 +72,8 @@ protected TokenStreamComponents createComponents(String fieldName) { @Override protected Reader initReader(String fieldName, Reader reader) { final AnalyzerComponents components = getComponents(); - if (components.charFilters != null && components.charFilters.length > 0) { - for (CharFilterFactory charFilter : components.charFilters) { + if (components.getCharFilters() != null && components.getCharFilters().length > 0) { + for (CharFilterFactory charFilter : components.getCharFilters()) { reader = charFilter.create(reader); } } @@ -112,7 +83,7 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected Reader initReaderForNormalization(String fieldName, Reader reader) { final AnalyzerComponents components = getComponents(); - for (CharFilterFactory charFilter : components.charFilters) { + for (CharFilterFactory charFilter : components.getCharFilters()) { reader = charFilter.normalize(reader); } return reader; @@ -122,7 +93,7 @@ protected Reader initReaderForNormalization(String fieldName, Reader reader) { protected TokenStream normalize(String fieldName, TokenStream in) { final AnalyzerComponents components = getComponents(); TokenStream result = in; - for (TokenFilterFactory filter : components.tokenFilters) { + for (TokenFilterFactory filter : components.getTokenFilters()) { result = filter.normalize(result); } return result; @@ -131,7 +102,7 @@ protected TokenStream normalize(String fieldName, TokenStream in) { private static AnalysisMode calculateAnalysisMode(AnalyzerComponents components) { // merge and transfer token filter analysis modes with analyzer AnalysisMode mode = AnalysisMode.ALL; - for (TokenFilterFactory f : components.tokenFilters) { + for (TokenFilterFactory f : components.getTokenFilters()) { mode = mode.merge(f.getAnalysisMode()); } return mode; diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 032ac16313168..60c9ed3263f3b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -107,13 +107,13 @@ public CustomAnalyzer get() { return this.customAnalyzer; } - static class AnalyzerComponents { - final String tokenizerName; - final TokenizerFactory tokenizerFactory; - final CharFilterFactory[] charFilters; - final TokenFilterFactory[] tokenFilters; - final int positionIncrementGap; - final int offsetGap; + public static class AnalyzerComponents { + private final String tokenizerName; + private final TokenizerFactory tokenizerFactory; + private final CharFilterFactory[] charFilters; + private final TokenFilterFactory[] tokenFilters; + private final int positionIncrementGap; + private final int offsetGap; AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, TokenFilterFactory[] tokenFilters, @@ -125,5 +125,29 @@ static class AnalyzerComponents { this.positionIncrementGap = positionIncrementGap; this.offsetGap = offsetGap; } + + public String getTokenizerName() { + return tokenizerName; + } + + public TokenizerFactory getTokenizerFactory() { + return tokenizerFactory; + } + + public TokenFilterFactory[] getTokenFilters() { + return tokenFilters; + } + + public CharFilterFactory[] getCharFilters() { + return charFilters; + } + + public int getPositionIncrementGap(String fieldName) { + return positionIncrementGap; + } + + public int getOffsetGap() { + return offsetGap; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java index 4831d88f3aa1f..bdadd92cb9dac 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java @@ -21,6 +21,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; +import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import org.elasticsearch.index.mapper.MapperException; import java.util.ArrayList; @@ -113,7 +114,8 @@ public void checkAllowedInMode(AnalysisMode mode) { } if (this.getAnalysisMode() != mode) { if (analyzer instanceof CustomAnalyzer) { - TokenFilterFactory[] tokenFilters = ((CustomAnalyzer) analyzer).tokenFilters(); + AnalyzerComponents components = ((CustomAnalyzer) analyzer).getComponents(); + TokenFilterFactory[] tokenFilters = components.getTokenFilters(); List offendingFilters = new ArrayList<>(); for (TokenFilterFactory tokenFilter : tokenFilters) { if (tokenFilter.getAnalysisMode() != mode) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java index 8506b2b5f427f..f3a62e90e1609 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java @@ -85,9 +85,9 @@ private AnalyzerComponents getStoredComponents() { @Override protected TokenStreamComponents createComponents(String fieldName) { final AnalyzerComponents components = getStoredComponents(); - Tokenizer tokenizer = components.tokenizerFactory.create(); + Tokenizer tokenizer = components.getTokenizerFactory().create(); TokenStream tokenStream = tokenizer; - for (TokenFilterFactory tokenFilter : components.tokenFilters) { + for (TokenFilterFactory tokenFilter : components.getTokenFilters()) { tokenStream = tokenFilter.create(tokenStream); } return new TokenStreamComponents(tokenizer, tokenStream); @@ -96,8 +96,8 @@ protected TokenStreamComponents createComponents(String fieldName) { @Override protected Reader initReader(String fieldName, Reader reader) { final AnalyzerComponents components = getStoredComponents(); - if (components.charFilters != null && components.charFilters.length > 0) { - for (CharFilterFactory charFilter : components.charFilters) { + if (components.getCharFilters() != null && components.getCharFilters().length > 0) { + for (CharFilterFactory charFilter : components.getCharFilters()) { reader = charFilter.create(reader); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index 583516c5cd4c2..d4f42ef217630 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -83,7 +83,7 @@ private static boolean containsBrokenAnalysis(Analyzer analyzer) { } if (analyzer instanceof CustomAnalyzer) { final CustomAnalyzer a = (CustomAnalyzer) analyzer; - TokenFilterFactory[] tokenFilters = a.tokenFilters(); + TokenFilterFactory[] tokenFilters = a.getComponents().getTokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { if (tokenFilterFactory.breaksFastVectorHighlighter()) { return true; diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index 74b9437d67821..b0c8d0e2e4abc 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -677,7 +677,7 @@ private static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyze } if (analyzer instanceof CustomAnalyzer) { final CustomAnalyzer a = (CustomAnalyzer) analyzer; - final TokenFilterFactory[] tokenFilters = a.tokenFilters(); + final TokenFilterFactory[] tokenFilters = a.getComponents().getTokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { if (tokenFilterFactory instanceof ShingleTokenFilterFactory) { return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index d4c2f40a8edac..d8a51515b8868 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -485,7 +485,7 @@ public void testReloadSearchAnalyzers() throws IOException { IndexAnalyzers current = mapperService.getIndexAnalyzers(); ReloadableCustomAnalyzer originalReloadableAnalyzer = (ReloadableCustomAnalyzer) current.get("reloadableAnalyzer").analyzer(); - TokenFilterFactory[] originalTokenFilters = originalReloadableAnalyzer.tokenFilters(); + TokenFilterFactory[] originalTokenFilters = originalReloadableAnalyzer.getComponents().getTokenFilters(); assertEquals(1, originalTokenFilters.length); assertEquals("myReloadableFilter", originalTokenFilters[0].name()); @@ -504,7 +504,7 @@ public void testReloadSearchAnalyzers() throws IOException { private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) { ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer(); - TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.tokenFilters(); + TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters(); assertEquals(originalTokenFilter.length, newTokenFilters.length); int i = 0; for (TokenFilterFactory tf : newTokenFilters ) { diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index a56834a4caf90..70fc4d6aec834 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -166,10 +166,10 @@ private void testSimpleConfiguration(Settings settings) throws IOException { assertThat(analyzer, instanceOf(CustomAnalyzer.class)); CustomAnalyzer custom1 = (CustomAnalyzer) analyzer; - assertThat(custom1.tokenizerFactory(), instanceOf(StandardTokenizerFactory.class)); - assertThat(custom1.tokenFilters().length, equalTo(2)); + assertThat(custom1.getComponents().getTokenizerFactory(), instanceOf(StandardTokenizerFactory.class)); + assertThat(custom1.getComponents().getTokenFilters().length, equalTo(2)); - StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0]; + StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.getComponents().getTokenFilters()[0]; assertThat(stop1.stopWords().size(), equalTo(1)); // verify position increment gap @@ -182,7 +182,7 @@ private void testSimpleConfiguration(Settings settings) throws IOException { analyzer = indexAnalyzers.get("custom4").analyzer(); assertThat(analyzer, instanceOf(CustomAnalyzer.class)); CustomAnalyzer custom4 = (CustomAnalyzer) analyzer; - assertThat(custom4.tokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class)); + assertThat(custom4.getComponents().getTokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class)); } public void testWordListPath() throws Exception { From ac82f936720fd17e92c97da28c59aa62eab1190b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 May 2019 19:12:33 +0200 Subject: [PATCH 16/20] Fix SynonymAnalyzerIT so it runs in gradle --- .../analysis/common/SynonymAnalyzerIT.java | 80 +++++++++---------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java index a8aad6f0921f7..5d4df091840ec 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java @@ -19,16 +19,16 @@ package org.elasticsearch.analysis.common; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; import org.elasticsearch.action.admin.indices.reloadanalyzer.ReloadAnalyzersResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.BeforeClass; +import org.elasticsearch.test.InternalTestCluster; import java.io.FileNotFoundException; import java.io.IOException; @@ -47,41 +47,32 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -@AwaitsFix(bugUrl="Cannot be run outside IDE yet") public class SynonymAnalyzerIT extends ESIntegTestCase { - private static Path config; - private static Path synonymsFile; - private static final String synonymsFileName = "synonyms.txt"; - - @BeforeClass - public static void initConfigDir() throws IOException { - config = createTempDir().resolve("config"); - if (Files.exists(config) == false) { - Files.createDirectory(config); - } - synonymsFile = config.resolve(synonymsFileName); - Files.createFile(synonymsFile); - assertTrue(Files.exists(synonymsFile)); - } - - @Override protected Collection> nodePlugins() { return Arrays.asList(CommonAnalysisPlugin.class); } + /** + * This test needs to write to the config directory, this is difficult in an external cluster so we overwrite this to force running with + * {@link InternalTestCluster} + */ @Override - protected Path nodeConfigPath(int nodeOrdinal) { - return config; + protected boolean ignoreExternalCluster() { + return true; } - public void testSynonymsUpdateable() throws FileNotFoundException, IOException { + public void testSynonymsUpdateable() throws FileNotFoundException, IOException, InterruptedException { + Path config = internalCluster().getInstance(Environment.class).configFile(); + String synonymsFileName = "synonyms.txt"; + Path synonymsFile = config.resolve(synonymsFileName); + Files.createFile(synonymsFile); + assertTrue(Files.exists(synonymsFile)); try (PrintWriter out = new PrintWriter( new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) { out.println("foo, baz"); } - assertTrue(Files.exists(synonymsFile)); assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() .put("index.number_of_shards", cluster().numDataNodes() * 2) .put("index.number_of_replicas", 1) @@ -104,26 +95,29 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException { assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm()); assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm()); - // now update synonyms file and trigger reloading - try (PrintWriter out = new PrintWriter( - new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { - out.println("foo, baz, buzz"); + // now update synonyms file several times and trigger reloading + for (int i = 0; i < 10; i++) { + String testTerm = randomAlphaOfLength(10); + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, " + testTerm); + } + ReloadAnalyzersResponse reloadResponse = client().admin().indices().prepareReloadAnalyzers("test").execute().actionGet(); + assertNoFailures(reloadResponse); + assertEquals(cluster().numDataNodes(), reloadResponse.getSuccessfulShards()); + + analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(3, analyzeResponse.getTokens().size()); + Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains(testTerm)); + + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", testTerm)).get(); + assertHitCount(response, 1L); } - ReloadAnalyzersResponse reloadResponse = client().admin().indices().prepareReloadAnalyzers("test").execute().actionGet(); - assertNoFailures(reloadResponse); - assertEquals(cluster().numDataNodes(), reloadResponse.getSuccessfulShards()); - - analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); - assertEquals(3, analyzeResponse.getTokens().size()); - Set tokens = new HashSet<>(); - analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); - assertTrue(tokens.contains("foo")); - assertTrue(tokens.contains("baz")); - assertTrue(tokens.contains("buzz")); - - response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); - assertHitCount(response, 1L); - response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); - assertHitCount(response, 1L); } } \ No newline at end of file From 5efac3ddfee74f40b3af879ba872aa48a5e726a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 23 May 2019 22:07:06 +0200 Subject: [PATCH 17/20] iter --- .../index/analysis/CustomAnalyzer.java | 13 +++++++++---- .../index/analysis/CustomAnalyzerProvider.java | 14 +++++++------- .../java/org/elasticsearch/test/ESTestCase.java | 1 - 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index ede17dfb08a22..d7f23c48b311b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -46,6 +46,11 @@ public AnalyzerComponents getComponents() { return this.components; } + @Override + public int getPositionIncrementGap(String fieldName) { + return this.components.getPositionIncrementGap(); + } + @Override public int getOffsetGap(String field) { if (this.components.getOffsetGap() < 0) { @@ -83,10 +88,10 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected Reader initReaderForNormalization(String fieldName, Reader reader) { final AnalyzerComponents components = getComponents(); - for (CharFilterFactory charFilter : components.getCharFilters()) { - reader = charFilter.normalize(reader); - } - return reader; + for (CharFilterFactory charFilter : components.getCharFilters()) { + reader = charFilter.normalize(reader); + } + return reader; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 60c9ed3263f3b..24a5094027ba6 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -50,11 +50,11 @@ void build(final Map tokenizers, customAnalyzer = CustomAnalyzer.create(components); } - static AnalyzerComponents createComponents(String name, Settings settings, + static AnalyzerComponents createComponents(String name, Settings analyzerSettings, final Map tokenizers, final Map charFilters, final Map tokenFilters) { - String tokenizerName = settings.get("tokenizer"); + String tokenizerName = analyzerSettings.get("tokenizer"); if (tokenizerName == null) { throw new IllegalArgumentException("Custom Analyzer [" + name + "] must be configured with a tokenizer"); } @@ -65,7 +65,7 @@ static AnalyzerComponents createComponents(String name, Settings settings, "[" + tokenizerName + "]"); } - List charFilterNames = settings.getAsList("char_filter"); + List charFilterNames = analyzerSettings.getAsList("char_filter"); List charFiltersList = new ArrayList<>(charFilterNames.size()); for (String charFilterName : charFilterNames) { CharFilterFactory charFilter = charFilters.get(charFilterName); @@ -78,11 +78,11 @@ static AnalyzerComponents createComponents(String name, Settings settings, int positionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP; - positionIncrementGap = settings.getAsInt("position_increment_gap", positionIncrementGap); + positionIncrementGap = analyzerSettings.getAsInt("position_increment_gap", positionIncrementGap); - int offsetGap = settings.getAsInt("offset_gap", -1); + int offsetGap = analyzerSettings.getAsInt("offset_gap", -1); - List tokenFilterNames = settings.getAsList("filter"); + List tokenFilterNames = analyzerSettings.getAsList("filter"); List tokenFilterList = new ArrayList<>(tokenFilterNames.size()); for (String tokenFilterName : tokenFilterNames) { TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName); @@ -142,7 +142,7 @@ public CharFilterFactory[] getCharFilters() { return charFilters; } - public int getPositionIncrementGap(String fieldName) { + public int getPositionIncrementGap() { return positionIncrementGap; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 2f5de3f3d5b05..6b36f985c210b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -29,7 +29,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; From 03fa79924095f51d628153329b125b35100f5511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 28 May 2019 20:19:49 +0200 Subject: [PATCH 18/20] Make ReloadableCustomAnalyzer sibling of CustomAnalyzer rather than extend it --- .../analyze/TransportAnalyzeAction.java | 40 +++++-- .../index/analysis/CustomAnalyzer.java | 112 +++++++++--------- .../analysis/CustomAnalyzerProvider.java | 36 +++++- .../index/analysis/NamedAnalyzer.java | 6 +- .../analysis/ReloadableCustomAnalyzer.java | 53 ++++++++- .../index/analysis/TokenFilterComposite.java | 28 +++++ .../highlight/FragmentBuilderHelper.java | 7 +- .../phrase/PhraseSuggestionBuilder.java | 7 +- .../index/mapper/MapperServiceTests.java | 7 -- .../indices/analysis/AnalysisModuleTests.java | 8 +- 10 files changed, 210 insertions(+), 94 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/TokenFilterComposite.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 9333c424da52c..133244e6659ac 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.NormalizingCharFilterFactory; import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; +import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -300,19 +301,36 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy } } - CustomAnalyzer customAnalyzer = null; - if (analyzer instanceof CustomAnalyzer) { - customAnalyzer = (CustomAnalyzer) analyzer; - } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { - customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer(); + Analyzer customAnalyzer = null; + // maybe unwrap analyzer from NamedAnalyzer + Analyzer potentialCustomAnalyzer = analyzer; + if (analyzer instanceof NamedAnalyzer) { + potentialCustomAnalyzer = ((NamedAnalyzer) analyzer).analyzer(); + } + if (potentialCustomAnalyzer instanceof CustomAnalyzer || potentialCustomAnalyzer instanceof ReloadableCustomAnalyzer) { + customAnalyzer = potentialCustomAnalyzer; } if (customAnalyzer != null) { - // customAnalyzer = divide charfilter, tokenizer tokenfilters - AnalyzerComponents components = customAnalyzer.getComponents(); - CharFilterFactory[] charFilterFactories = components.getCharFilters(); - TokenizerFactory tokenizerFactory = components.getTokenizerFactory(); - TokenFilterFactory[] tokenFilterFactories = components.getTokenFilters(); + // divide charfilter, tokenizer tokenfilters + CharFilterFactory[] charFilterFactories; + TokenizerFactory tokenizerFactory; + TokenFilterFactory[] tokenFilterFactories; + String tokenizerName; + if (customAnalyzer instanceof CustomAnalyzer) { + CustomAnalyzer casted = (CustomAnalyzer) analyzer; + charFilterFactories = casted.charFilters(); + tokenizerFactory = casted.tokenizerFactory(); + tokenFilterFactories = casted.tokenFilters(); + tokenizerName = casted.getTokenizerName(); + } else { + // for ReloadableCustomAnalyzer we want to make sure we get the factories from the same components object + AnalyzerComponents components = ((ReloadableCustomAnalyzer) customAnalyzer).getComponents(); + charFilterFactories = components.getCharFilters(); + tokenizerFactory = components.getTokenizerFactory(); + tokenFilterFactories = components.getTokenFilters(); + tokenizerName = components.getTokenizerName(); + } String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length]; TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? @@ -372,7 +390,7 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy } } detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList( - components.getTokenizerName(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); + tokenizerName, tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); } else { String name; if (analyzer instanceof NamedAnalyzer) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index d7f23c48b311b..2c6622cf6b532 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -22,41 +22,73 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; -import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import java.io.Reader; -public class CustomAnalyzer extends Analyzer { +public final class CustomAnalyzer extends Analyzer implements TokenFilterComposite { + private final String tokenizerName; + private final TokenizerFactory tokenizerFactory; + + private final CharFilterFactory[] charFilters; + + private final TokenFilterFactory[] tokenFilters; + + private final int positionIncrementGap; + private final int offsetGap; private final AnalysisMode analysisMode; - protected volatile AnalyzerComponents components; public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, - TokenFilterFactory[] tokenFilters) { - this(new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters, 0, -1), GLOBAL_REUSE_STRATEGY); + TokenFilterFactory[] tokenFilters) { + this(tokenizerName, tokenizerFactory, charFilters, tokenFilters, 0, -1); + } + + public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, + TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) { + this.tokenizerName = tokenizerName; + this.tokenizerFactory = tokenizerFactory; + this.charFilters = charFilters; + this.tokenFilters = tokenFilters; + this.positionIncrementGap = positionIncrementGap; + this.offsetGap = offsetGap; + // merge and transfer token filter analysis modes with analyzer + AnalysisMode mode = AnalysisMode.ALL; + for (TokenFilterFactory f : tokenFilters) { + mode = mode.merge(f.getAnalysisMode()); + } + this.analysisMode = mode; + } + + /** + * The name of the tokenizer as configured by the user. + */ + public String getTokenizerName() { + return tokenizerName; + } + + public TokenizerFactory tokenizerFactory() { + return tokenizerFactory; } - CustomAnalyzer(AnalyzerComponents components, ReuseStrategy reuseStrategy) { - super(reuseStrategy); - this.components = components; - this.analysisMode = calculateAnalysisMode(components); + public TokenFilterFactory[] tokenFilters() { + return tokenFilters; } - public AnalyzerComponents getComponents() { - return this.components; + public CharFilterFactory[] charFilters() { + return charFilters; } @Override public int getPositionIncrementGap(String fieldName) { - return this.components.getPositionIncrementGap(); + return this.positionIncrementGap; } @Override public int getOffsetGap(String field) { - if (this.components.getOffsetGap() < 0) { + if (offsetGap < 0) { return super.getOffsetGap(field); } - return this.components.getOffsetGap(); + return this.offsetGap; } public AnalysisMode getAnalysisMode() { @@ -65,10 +97,9 @@ public AnalysisMode getAnalysisMode() { @Override protected TokenStreamComponents createComponents(String fieldName) { - final AnalyzerComponents components = getComponents(); - Tokenizer tokenizer = components.getTokenizerFactory().create(); + Tokenizer tokenizer = tokenizerFactory.create(); TokenStream tokenStream = tokenizer; - for (TokenFilterFactory tokenFilter : components.getTokenFilters()) { + for (TokenFilterFactory tokenFilter : tokenFilters) { tokenStream = tokenFilter.create(tokenStream); } return new TokenStreamComponents(tokenizer, tokenStream); @@ -76,9 +107,8 @@ protected TokenStreamComponents createComponents(String fieldName) { @Override protected Reader initReader(String fieldName, Reader reader) { - final AnalyzerComponents components = getComponents(); - if (components.getCharFilters() != null && components.getCharFilters().length > 0) { - for (CharFilterFactory charFilter : components.getCharFilters()) { + if (charFilters != null && charFilters.length > 0) { + for (CharFilterFactory charFilter : charFilters) { reader = charFilter.create(reader); } } @@ -87,42 +117,18 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected Reader initReaderForNormalization(String fieldName, Reader reader) { - final AnalyzerComponents components = getComponents(); - for (CharFilterFactory charFilter : components.getCharFilters()) { - reader = charFilter.normalize(reader); - } - return reader; + for (CharFilterFactory charFilter : charFilters) { + reader = charFilter.normalize(reader); + } + return reader; } @Override protected TokenStream normalize(String fieldName, TokenStream in) { - final AnalyzerComponents components = getComponents(); - TokenStream result = in; - for (TokenFilterFactory filter : components.getTokenFilters()) { - result = filter.normalize(result); - } - return result; - } - - private static AnalysisMode calculateAnalysisMode(AnalyzerComponents components) { - // merge and transfer token filter analysis modes with analyzer - AnalysisMode mode = AnalysisMode.ALL; - for (TokenFilterFactory f : components.getTokenFilters()) { - mode = mode.merge(f.getAnalysisMode()); - } - return mode; - } - - /** - * Factory method that either returns a plain {@link CustomAnalyzer} if the components used for creation are supporting index and search - * time use, or a {@link ReloadableCustomAnalyzer} if the components are intended for search time use only. - */ - static CustomAnalyzer create(AnalyzerComponents components) { - AnalysisMode mode = calculateAnalysisMode(components); - if (mode.equals(AnalysisMode.SEARCH_TIME)) { - return new ReloadableCustomAnalyzer(components); - } else { - return new CustomAnalyzer(components, GLOBAL_REUSE_STRATEGY); - } + TokenStream result = in; + for (TokenFilterFactory filter : tokenFilters) { + result = filter.normalize(result); + } + return result; } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 24a5094027ba6..290571d711c81 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.analysis; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -31,11 +32,11 @@ * A custom analyzer that is built out of a single {@link org.apache.lucene.analysis.Tokenizer} and a list * of {@link org.apache.lucene.analysis.TokenFilter}s. */ -public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider { +public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider { private final Settings analyzerSettings; - private CustomAnalyzer customAnalyzer; + private Analyzer customAnalyzer; public CustomAnalyzerProvider(IndexSettings indexSettings, String name, Settings settings) { @@ -46,8 +47,23 @@ public CustomAnalyzerProvider(IndexSettings indexSettings, void build(final Map tokenizers, final Map charFilters, final Map tokenFilters) { - AnalyzerComponents components = createComponents(name(), analyzerSettings, tokenizers, charFilters, tokenFilters); - customAnalyzer = CustomAnalyzer.create(components); + customAnalyzer = create(name(), analyzerSettings, tokenizers, charFilters, tokenFilters); + } + + /** + * Factory method that either returns a plain {@link ReloadableCustomAnalyzer} if the components used for creation are supporting index + * and search time use, or a {@link ReloadableCustomAnalyzer} if the components are intended for search time use only. + */ + private static Analyzer create(String name, Settings analyzerSettings, Map tokenizers, + Map charFilters, + Map tokenFilters) { + AnalyzerComponents components = createComponents(name, analyzerSettings, tokenizers, charFilters, tokenFilters); + if (components.analysisMode().equals(AnalysisMode.SEARCH_TIME)) { + return new ReloadableCustomAnalyzer(components); + } else { + return new CustomAnalyzer(components.getTokenizerName(), components.getTokenizerFactory(), components.getCharFilters(), + components.getTokenFilters(), components.getPositionIncrementGap(), components.getOffsetGap()); + } } static AnalyzerComponents createComponents(String name, Settings analyzerSettings, @@ -103,7 +119,7 @@ static AnalyzerComponents createComponents(String name, Settings analyzerSetting } @Override - public CustomAnalyzer get() { + public Analyzer get() { return this.customAnalyzer; } @@ -114,6 +130,7 @@ public static class AnalyzerComponents { private final TokenFilterFactory[] tokenFilters; private final int positionIncrementGap; private final int offsetGap; + private final AnalysisMode analysisMode; AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, TokenFilterFactory[] tokenFilters, @@ -124,6 +141,11 @@ public static class AnalyzerComponents { this.tokenFilters = tokenFilters; this.positionIncrementGap = positionIncrementGap; this.offsetGap = offsetGap; + AnalysisMode mode = AnalysisMode.ALL; + for (TokenFilterFactory f : tokenFilters) { + mode = mode.merge(f.getAnalysisMode()); + } + this.analysisMode = mode; } public String getTokenizerName() { @@ -149,5 +171,9 @@ public int getPositionIncrementGap() { public int getOffsetGap() { return offsetGap; } + + public AnalysisMode analysisMode() { + return this.analysisMode; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java index bdadd92cb9dac..b2c786472fd5c 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java @@ -21,7 +21,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; -import org.elasticsearch.index.analysis.CustomAnalyzerProvider.AnalyzerComponents; import org.elasticsearch.index.mapper.MapperException; import java.util.ArrayList; @@ -113,9 +112,8 @@ public void checkAllowedInMode(AnalysisMode mode) { return; // everything allowed if this analyzer is in ALL mode } if (this.getAnalysisMode() != mode) { - if (analyzer instanceof CustomAnalyzer) { - AnalyzerComponents components = ((CustomAnalyzer) analyzer).getComponents(); - TokenFilterFactory[] tokenFilters = components.getTokenFilters(); + if (analyzer instanceof TokenFilterComposite) { + TokenFilterFactory[] tokenFilters = ((TokenFilterComposite) analyzer).tokenFilters(); List offendingFilters = new ArrayList<>(); for (TokenFilterFactory tokenFilter : tokenFilters) { if (tokenFilter.getAnalysisMode() != mode) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java index f3a62e90e1609..1bd14b8d3d34a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java @@ -29,7 +29,10 @@ import java.io.Reader; import java.util.Map; -public final class ReloadableCustomAnalyzer extends CustomAnalyzer { +public class ReloadableCustomAnalyzer extends Analyzer implements TokenFilterComposite { + + private volatile AnalyzerComponents components; + private CloseableThreadLocal storedComponents = new CloseableThreadLocal<>(); private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { @@ -53,7 +56,53 @@ public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStre }; ReloadableCustomAnalyzer(AnalyzerComponents components) { - super(components, UPDATE_STRATEGY); + super(UPDATE_STRATEGY); + this.components = components; + } + + public AnalyzerComponents getComponents() { + return this.components; + } + + @Override + public TokenFilterFactory[] tokenFilters() { + return this.components.getTokenFilters(); + } + + @Override + public int getPositionIncrementGap(String fieldName) { + return this.components.getPositionIncrementGap(); + } + + @Override + public int getOffsetGap(String field) { + if (this.components.getOffsetGap() < 0) { + return super.getOffsetGap(field); + } + return this.components.getOffsetGap(); + } + + public AnalysisMode getAnalysisMode() { + return this.components.analysisMode(); + } + + @Override + protected Reader initReaderForNormalization(String fieldName, Reader reader) { + final AnalyzerComponents components = getComponents(); + for (CharFilterFactory charFilter : components.getCharFilters()) { + reader = charFilter.normalize(reader); + } + return reader; + } + + @Override + protected TokenStream normalize(String fieldName, TokenStream in) { + final AnalyzerComponents components = getComponents(); + TokenStream result = in; + for (TokenFilterFactory filter : components.getTokenFilters()) { + result = filter.normalize(result); + } + return result; } private boolean shouldReload(AnalyzerComponents source) { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterComposite.java b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterComposite.java new file mode 100644 index 0000000000000..94ca50ba38745 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterComposite.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +/** + * Analyzers that provide access to their token filters should implement this + */ +public interface TokenFilterComposite { + + TokenFilterFactory[] tokenFilters(); +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index d4f42ef217630..813a32ff03c0b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -26,8 +26,8 @@ import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; import org.apache.lucene.search.vectorhighlight.FragmentsBuilder; import org.apache.lucene.util.CollectionUtil; -import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.TokenFilterComposite; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.MappedFieldType; @@ -81,9 +81,8 @@ private static boolean containsBrokenAnalysis(Analyzer analyzer) { if (analyzer instanceof NamedAnalyzer) { analyzer = ((NamedAnalyzer) analyzer).analyzer(); } - if (analyzer instanceof CustomAnalyzer) { - final CustomAnalyzer a = (CustomAnalyzer) analyzer; - TokenFilterFactory[] tokenFilters = a.getComponents().getTokenFilters(); + if (analyzer instanceof TokenFilterComposite) { + final TokenFilterFactory[] tokenFilters = ((TokenFilterComposite) analyzer).tokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { if (tokenFilterFactory.breaksFastVectorHighlighter()) { return true; diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index b0c8d0e2e4abc..2b1b395c574de 100644 --- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -31,9 +31,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterComposite; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardContext; @@ -675,9 +675,8 @@ private static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyze if (analyzer instanceof NamedAnalyzer) { analyzer = ((NamedAnalyzer)analyzer).analyzer(); } - if (analyzer instanceof CustomAnalyzer) { - final CustomAnalyzer a = (CustomAnalyzer) analyzer; - final TokenFilterFactory[] tokenFilters = a.getComponents().getTokenFilters(); + if (analyzer instanceof TokenFilterComposite) { + final TokenFilterFactory[] tokenFilters = ((TokenFilterComposite) analyzer).tokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { if (tokenFilterFactory instanceof ShingleTokenFilterFactory) { return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index d8a51515b8868..6bdfc167dec8b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -448,13 +448,6 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName()); } - public void testReloadSearchAnalyzersNoReload() throws IOException { - MapperService mapperService = createIndex("no_reload_index", Settings.EMPTY).mapperService(); - IndexAnalyzers current = mapperService.getIndexAnalyzers(); - mapperService.reloadSearchAnalyzers(getInstanceFromNode(AnalysisRegistry.class)); - assertSame(current, mapperService.getIndexAnalyzers()); - } - public void testReloadSearchAnalyzers() throws IOException { Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index 70fc4d6aec834..a56834a4caf90 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -166,10 +166,10 @@ private void testSimpleConfiguration(Settings settings) throws IOException { assertThat(analyzer, instanceOf(CustomAnalyzer.class)); CustomAnalyzer custom1 = (CustomAnalyzer) analyzer; - assertThat(custom1.getComponents().getTokenizerFactory(), instanceOf(StandardTokenizerFactory.class)); - assertThat(custom1.getComponents().getTokenFilters().length, equalTo(2)); + assertThat(custom1.tokenizerFactory(), instanceOf(StandardTokenizerFactory.class)); + assertThat(custom1.tokenFilters().length, equalTo(2)); - StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.getComponents().getTokenFilters()[0]; + StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0]; assertThat(stop1.stopWords().size(), equalTo(1)); // verify position increment gap @@ -182,7 +182,7 @@ private void testSimpleConfiguration(Settings settings) throws IOException { analyzer = indexAnalyzers.get("custom4").analyzer(); assertThat(analyzer, instanceOf(CustomAnalyzer.class)); CustomAnalyzer custom4 = (CustomAnalyzer) analyzer; - assertThat(custom4.getComponents().getTokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class)); + assertThat(custom4.tokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class)); } public void testWordListPath() throws Exception { From 6d120842a4c8e70ca898707f5e7a4d7d9704fb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 28 May 2019 22:12:00 +0200 Subject: [PATCH 19/20] Pull positionIncrement and offsetGap out of AnalyzerComponents --- .../analysis/CustomAnalyzerProvider.java | 19 +++++-------------- .../analysis/ReloadableCustomAnalyzer.java | 14 ++++++++++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index 290571d711c81..b6ed790cc838e 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -57,12 +57,15 @@ void build(final Map tokenizers, private static Analyzer create(String name, Settings analyzerSettings, Map tokenizers, Map charFilters, Map tokenFilters) { + int positionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP; + positionIncrementGap = analyzerSettings.getAsInt("position_increment_gap", positionIncrementGap); + int offsetGap = analyzerSettings.getAsInt("offset_gap", -1); AnalyzerComponents components = createComponents(name, analyzerSettings, tokenizers, charFilters, tokenFilters); if (components.analysisMode().equals(AnalysisMode.SEARCH_TIME)) { - return new ReloadableCustomAnalyzer(components); + return new ReloadableCustomAnalyzer(components, positionIncrementGap, offsetGap); } else { return new CustomAnalyzer(components.getTokenizerName(), components.getTokenizerFactory(), components.getCharFilters(), - components.getTokenFilters(), components.getPositionIncrementGap(), components.getOffsetGap()); + components.getTokenFilters(), positionIncrementGap, offsetGap); } } @@ -128,8 +131,6 @@ public static class AnalyzerComponents { private final TokenizerFactory tokenizerFactory; private final CharFilterFactory[] charFilters; private final TokenFilterFactory[] tokenFilters; - private final int positionIncrementGap; - private final int offsetGap; private final AnalysisMode analysisMode; AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, @@ -139,8 +140,6 @@ public static class AnalyzerComponents { this.tokenizerFactory = tokenizerFactory; this.charFilters = charFilters; this.tokenFilters = tokenFilters; - this.positionIncrementGap = positionIncrementGap; - this.offsetGap = offsetGap; AnalysisMode mode = AnalysisMode.ALL; for (TokenFilterFactory f : tokenFilters) { mode = mode.merge(f.getAnalysisMode()); @@ -164,14 +163,6 @@ public CharFilterFactory[] getCharFilters() { return charFilters; } - public int getPositionIncrementGap() { - return positionIncrementGap; - } - - public int getOffsetGap() { - return offsetGap; - } - public AnalysisMode analysisMode() { return this.analysisMode; } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java index 1bd14b8d3d34a..0c65440d1372d 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java @@ -35,6 +35,10 @@ public class ReloadableCustomAnalyzer extends Analyzer implements TokenFilterCom private CloseableThreadLocal storedComponents = new CloseableThreadLocal<>(); + private final int positionIncrementGap; + + private final int offsetGap; + private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { @Override public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) { @@ -55,9 +59,11 @@ public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStre } }; - ReloadableCustomAnalyzer(AnalyzerComponents components) { + ReloadableCustomAnalyzer(AnalyzerComponents components, int positionIncrementGap, int offsetGap) { super(UPDATE_STRATEGY); this.components = components; + this.positionIncrementGap = positionIncrementGap; + this.offsetGap = offsetGap; } public AnalyzerComponents getComponents() { @@ -71,15 +77,15 @@ public TokenFilterFactory[] tokenFilters() { @Override public int getPositionIncrementGap(String fieldName) { - return this.components.getPositionIncrementGap(); + return this.positionIncrementGap; } @Override public int getOffsetGap(String field) { - if (this.components.getOffsetGap() < 0) { + if (this.offsetGap < 0) { return super.getOffsetGap(field); } - return this.components.getOffsetGap(); + return this.offsetGap; } public AnalysisMode getAnalysisMode() { From 4affb0b3c9281b76bfa9621fe8a2a161b50237ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 29 May 2019 14:42:24 +0200 Subject: [PATCH 20/20] iter --- .../reloadanalyzer/TransportReloadAnalyzersAction.java | 1 - .../index/analysis/CustomAnalyzerProvider.java | 7 ++----- .../index/analysis/ReloadableCustomAnalyzer.java | 6 +++++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java index 6c352996d4761..b2b4ad3e1b02c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/reloadanalyzer/TransportReloadAnalyzersAction.java @@ -154,7 +154,6 @@ protected ShardsIterator shards(ClusterState clusterState, ReloadAnalyzersReques } } } - logger.info("Determined shards for reloading: " + shards); return new PlainShardsIterator(shards); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java index b6ed790cc838e..87f1fc254c7e6 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java @@ -115,9 +115,7 @@ static AnalyzerComponents createComponents(String name, Settings analyzerSetting return new AnalyzerComponents(tokenizerName, tokenizer, charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]), - tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]), - positionIncrementGap, - offsetGap + tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]) ); } @@ -134,8 +132,7 @@ public static class AnalyzerComponents { private final AnalysisMode analysisMode; AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters, - TokenFilterFactory[] tokenFilters, - int positionIncrementGap, int offsetGap) { + TokenFilterFactory[] tokenFilters) { this.tokenizerName = tokenizerName; this.tokenizerFactory = tokenizerFactory; this.charFilters = charFilters; diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java index 0c65440d1372d..2fbbf5b217fa0 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java @@ -29,7 +29,7 @@ import java.io.Reader; import java.util.Map; -public class ReloadableCustomAnalyzer extends Analyzer implements TokenFilterComposite { +public final class ReloadableCustomAnalyzer extends Analyzer implements TokenFilterComposite { private volatile AnalyzerComponents components; @@ -39,6 +39,10 @@ public class ReloadableCustomAnalyzer extends Analyzer implements TokenFilterCom private final int offsetGap; + /** + * An alternative {@link ReuseStrategy} that allows swapping the stored the analyzer components when they change. + * This is used to change e.g. token filters in search time analyzers. + */ private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() { @Override public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {