diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java index 75d4eca4254f8..5d6135549b882 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; +import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; @@ -50,6 +51,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { private final boolean lenient; protected final Settings settings; protected final Environment environment; + private final boolean updateable; SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { @@ -65,9 +67,15 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { this.expand = settings.getAsBoolean("expand", true); this.lenient = settings.getAsBoolean("lenient", false); this.format = settings.get("format", ""); + this.updateable = settings.getAsBoolean("updateable", false); this.environment = env; } + @Override + public AnalysisMode getAnalysisMode() { + return this.updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; + } + @Override public TokenStream create(TokenStream tokenStream) { throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first"); @@ -98,6 +106,11 @@ public TokenFilterFactory getSynonymFilter() { // which doesn't support stacked input tokens return IDENTITY_FILTER; } + + @Override + public AnalysisMode getAnalysisMode() { + return updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; + } }; } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java new file mode 100644 index 0000000000000..2c6d66720d390 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +@AwaitsFix(bugUrl="Cannot be run outside IDE yet") +public class SynonymAnalyzerIT extends ESIntegTestCase { + + private static Path config; + private static Path synonymsFile; + private static final String synonymsFileName = "synonyms.txt"; + + @BeforeClass + public static void initConfigDir() throws IOException { + config = createTempDir().resolve("config"); + if (Files.exists(config) == false) { + Files.createDirectory(config); + } + synonymsFile = config.resolve(synonymsFileName); + Files.createFile(synonymsFile); + assertTrue(Files.exists(synonymsFile)); + } + + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + @Override + protected Path nodeConfigPath(int nodeOrdinal) { + return config; + } + + public void testSynonymsUpdateable() throws FileNotFoundException, IOException { + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) { + out.println("foo, baz"); + } + assertTrue(Files.exists(synonymsFile)); + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") + .put("analysis.analyzer.my_synonym_analyzer.filter", "my_synonym_filter") + .put("analysis.filter.my_synonym_filter.type", "synonym") + .put("analysis.filter.my_synonym_filter.updateable", "true") + .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName)) + .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer")); + + client().prepareIndex("test", "_doc", "1").setSource("field", "foo").get(); + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 0L); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(2, analyzeResponse.getTokens().size()); + assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm()); + assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm()); + + // now update synonyms file and trigger reloading + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, buzz"); + } + // TODO don't use refresh here but something more specific + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(3, analyzeResponse.getTokens().size()); + Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains("buzz")); + + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 1L); + } +} \ No newline at end of file diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java new file mode 100644 index 0000000000000..9175999bb687b --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +public class SynonymAnalyzerTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + public void testSynonymsUpdateable() throws FileNotFoundException, IOException { + String synonymsFileName = "synonyms.txt"; + Path configDir = node().getEnvironment().configFile(); + if (Files.exists(configDir) == false) { + Files.createDirectory(configDir); + } + Path synonymsFile = configDir.resolve(synonymsFileName); + if (Files.exists(synonymsFile) == false) { + Files.createFile(synonymsFile); + } + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz"); + } + + assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard") + .putList("analysis.analyzer.my_synonym_analyzer.filter", "lowercase", "my_synonym_filter") + .put("analysis.filter.my_synonym_filter.type", "synonym") + .put("analysis.filter.my_synonym_filter.updateable", "true") + .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName)) + .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer")); + + client().prepareIndex("test", "_doc", "1").setSource("field", "Foo").get(); + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 0L); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(2, analyzeResponse.getTokens().size()); + assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm()); + assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm()); + + // now update synonyms file and trigger reloading + try (PrintWriter out = new PrintWriter( + new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) { + out.println("foo, baz, buzz"); + } + // TODO don't use refresh here but something more specific + assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet()); + + analyzeResponse = client().admin().indices().prepareAnalyze("test", "Foo").setAnalyzer("my_synonym_analyzer").get(); + assertEquals(3, analyzeResponse.getTokens().size()); + Set tokens = new HashSet<>(); + analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t)); + assertTrue(tokens.contains("foo")); + assertTrue(tokens.contains("baz")); + assertTrue(tokens.contains("buzz")); + + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get(); + assertHitCount(response, 1L); + response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get(); + assertHitCount(response, 1L); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index c0a52ac8c0d6a..2a30bd11e6566 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -58,6 +58,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind ActionListener> listener) { ActionListener.completeWith(listener, () -> { primary.refresh("api"); + try { + primary.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage(), ex); + return new PrimaryResult(null, null, ex); + } logger.trace("{} refresh request executed on primary", primary.shardId()); return new PrimaryResult<>(shardRequest, new ReplicationResponse()); }); @@ -66,6 +72,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind @Override protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica) { replica.refresh("api"); + try { + replica.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis()); + } catch (Exception ex) { + logger.error(ex.getLocalizedMessage(), ex); + return new ReplicaResult(ex); + } logger.trace("{} refresh request executed on replica", replica.shardId()); return new ReplicaResult(); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java index 483a1b4a7e563..674bca2ca2726 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; @@ -34,11 +33,14 @@ import java.io.Closeable; import java.io.IOException; +import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Collections.unmodifiableMap; @@ -151,13 +153,12 @@ public void close() throws IOException { * Creates an index-level {@link IndexAnalyzers} from this registry using the given index settings */ public IndexAnalyzers build(IndexSettings indexSettings) throws IOException { - final Map charFilterFactories = buildCharFilterFactories(indexSettings); final Map tokenizerFactories = buildTokenizerFactories(indexSettings); final Map tokenFilterFactories = buildTokenFilterFactories(indexSettings); - final Map> analyzierFactories = buildAnalyzerFactories(indexSettings); + final Map> analyzerFactories = buildAnalyzerFactories(indexSettings); final Map> normalizerFactories = buildNormalizerFactories(indexSettings); - return build(indexSettings, analyzierFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); + return build(indexSettings, analyzerFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories); } public Map buildTokenFilterFactories(IndexSettings indexSettings) throws IOException { @@ -197,13 +198,21 @@ public Map> buildNormalizerFactories(IndexSettings i * @return {@link TokenizerFactory} provider or null */ public AnalysisProvider getTokenizerProvider(String tokenizer, IndexSettings indexSettings) { - final Map tokenizerSettings = indexSettings.getSettings().getGroups("index.analysis.tokenizer"); - if (tokenizerSettings.containsKey(tokenizer)) { - Settings currentSettings = tokenizerSettings.get(tokenizer); - return getAnalysisProvider(Component.TOKENIZER, tokenizers, tokenizer, currentSettings.get("type")); - } else { - return getTokenizerProvider(tokenizer); - } + return getProvider(Component.TOKENIZER, tokenizer, indexSettings, "index.analysis.tokenizer", tokenizers, + this::getTokenizerProvider); + } + + /** + * Returns a registered {@link CharFilterFactory} provider by {@link IndexSettings} + * or a registered {@link CharFilterFactory} provider by predefined name + * or null if the charFilter was not registered + * @param charFilter global or defined charFilter name + * @param indexSettings an index settings + * @return {@link CharFilterFactory} provider or null + */ + public AnalysisProvider getCharFilterProvider(String charFilter, IndexSettings indexSettings) { + return getProvider(Component.CHAR_FILTER, charFilter, indexSettings, "index.analysis.char_filter", charFilters, + this::getCharFilterProvider); } /** @@ -215,31 +224,18 @@ public AnalysisProvider getTokenizerProvider(String tokenizer, * @return {@link TokenFilterFactory} provider or null */ public AnalysisProvider getTokenFilterProvider(String tokenFilter, IndexSettings indexSettings) { - final Map tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.filter"); - if (tokenFilterSettings.containsKey(tokenFilter)) { - Settings currentSettings = tokenFilterSettings.get(tokenFilter); - String typeName = currentSettings.get("type"); - return getAnalysisProvider(Component.FILTER, tokenFilters, tokenFilter, typeName); - } else { - return getTokenFilterProvider(tokenFilter); - } + return getProvider(Component.FILTER, tokenFilter, indexSettings, "index.analysis.filter", tokenFilters, + this::getTokenFilterProvider); } - /** - * Returns a registered {@link CharFilterFactory} provider by {@link IndexSettings} - * or a registered {@link CharFilterFactory} provider by predefined name - * or null if the charFilter was not registered - * @param charFilter global or defined charFilter name - * @param indexSettings an index settings - * @return {@link CharFilterFactory} provider or null - */ - public AnalysisProvider getCharFilterProvider(String charFilter, IndexSettings indexSettings) { - final Map tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.char_filter"); - if (tokenFilterSettings.containsKey(charFilter)) { - Settings currentSettings = tokenFilterSettings.get(charFilter); - return getAnalysisProvider(Component.CHAR_FILTER, charFilters, charFilter, currentSettings.get("type")); + private AnalysisProvider getProvider(Component componentType, String componentName, IndexSettings indexSettings, + String componentSettings, Map> providers, Function> providerFunction) { + final Map subSettings = indexSettings.getSettings().getGroups(componentSettings); + if (subSettings.containsKey(componentName)) { + Settings currentSettings = subSettings.get(componentName); + return getAnalysisProvider(componentType, providers, componentName, currentSettings.get("type")); } else { - return getCharFilterProvider(charFilter); + return providerFunction.apply(componentName); } } @@ -277,7 +273,7 @@ public String toString() { } @SuppressWarnings("unchecked") - private Map buildMapping(Component component, IndexSettings settings, Map settingsMap, + Map buildMapping(Component component, IndexSettings settings, Map settingsMap, Map> providerMap, Map> defaultInstance) throws IOException { Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build(); @@ -318,9 +314,9 @@ private Map buildMapping(Component component, IndexSettings setti } // go over the char filters in the bindings and register the ones that are not configured - for (Map.Entry> entry : providerMap.entrySet()) { + for (Map.Entry> entry : providerMap.entrySet()) { String name = entry.getKey(); - AnalysisModule.AnalysisProvider provider = entry.getValue(); + AnalysisProvider provider = entry.getValue(); // we don't want to re-register one that already exists if (settingsMap.containsKey(name)) { continue; @@ -342,17 +338,12 @@ private Map buildMapping(Component component, IndexSettings setti for (Map.Entry> entry : defaultInstance.entrySet()) { final String name = entry.getKey(); final AnalysisModule.AnalysisProvider provider = entry.getValue(); - if (factories.containsKey(name) == false) { - final T instance = provider.get(settings, environment, name, defaultSettings); - if (factories.containsKey(name) == false) { - factories.put(name, instance); - } - } + factories.putIfAbsent(name, provider.get(settings, environment, name, defaultSettings)); } return factories; } - private AnalysisProvider getAnalysisProvider(Component component, Map> providerMap, + private static AnalysisProvider getAnalysisProvider(Component component, Map> providerMap, String name, String typeName) { if (typeName == null) { throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer"); @@ -364,9 +355,9 @@ private AnalysisProvider getAnalysisProvider(Component component, Map>> analyzerProviderFactories; + final Map>> analyzerProviderFactories; final Map> preConfiguredTokenFilters; final Map> preConfiguredTokenizers; final Map> preConfiguredCharFilterFactories; @@ -391,19 +382,19 @@ private PrebuiltAnalysis( this.preConfiguredTokenizers = preConfiguredTokenizers; } - public AnalysisModule.AnalysisProvider getCharFilterFactory(String name) { + public AnalysisProvider getCharFilterFactory(String name) { return preConfiguredCharFilterFactories.get(name); } - public AnalysisModule.AnalysisProvider getTokenFilterFactory(String name) { + public AnalysisProvider getTokenFilterFactory(String name) { return preConfiguredTokenFilters.get(name); } - public AnalysisModule.AnalysisProvider getTokenizerFactory(String name) { + public AnalysisProvider getTokenizerFactory(String name) { return preConfiguredTokenizers.get(name); } - public AnalysisModule.AnalysisProvider> getAnalyzerProvider(String name) { + public AnalysisProvider> getAnalyzerProvider(String name) { return analyzerProviderFactories.get(name); } @@ -420,9 +411,6 @@ public IndexAnalyzers build(IndexSettings indexSettings, Map tokenizerFactoryFactories, Map charFilterFactoryFactories, Map tokenFilterFactoryFactories) { - - Index index = indexSettings.getIndex(); - analyzerProviders = new HashMap<>(analyzerProviders); Map analyzers = new HashMap<>(); Map normalizers = new HashMap<>(); Map whitespaceNormalizers = new HashMap<>(); @@ -458,7 +446,7 @@ public IndexAnalyzers build(IndexSettings indexSettings, defaultAnalyzer.checkAllowedInMode(AnalysisMode.ALL); if (analyzers.containsKey("default_index")) { throw new IllegalArgumentException("setting [index.analysis.analyzer.default_index] is not supported anymore, use " + - "[index.analysis.analyzer.default] instead for index [" + index.getName() + "]"); + "[index.analysis.analyzer.default] instead for index [" + indexSettings.getIndex().getName() + "]"); } NamedAnalyzer defaultSearchAnalyzer = analyzers.getOrDefault("default_search", defaultAnalyzer); NamedAnalyzer defaultSearchQuoteAnalyzer = analyzers.getOrDefault("default_search_quote", defaultSearchAnalyzer); @@ -472,9 +460,11 @@ public IndexAnalyzers build(IndexSettings indexSettings, whitespaceNormalizers); } - private static NamedAnalyzer produceAnalyzer(String name, AnalyzerProvider analyzerFactory, - Map tokenFilters, Map charFilters, - Map tokenizers) { + private static NamedAnalyzer produceAnalyzer(String name, + AnalyzerProvider analyzerFactory, + Map tokenFilters, + Map charFilters, + Map tokenizers) { /* * Lucene defaults positionIncrementGap to 0 in all analyzers but * Elasticsearch defaults them to 0 only before version 2.0 @@ -536,4 +526,80 @@ private void processNormalizerFactory( } normalizers.put(name, normalizer); } + + /** + * Create an new IndexAnalyzer instance based on the existing one. Analyzers that are in {@link AnalysisMode#SEARCH_TIME} are tried to + * be reloaded. All other analyzers are reused from the old {@link IndexAnalyzers} instance. + */ + public IndexAnalyzers rebuildIndexAnalyzers(IndexAnalyzers indexAnalyzers, IndexSettings indexSettings) throws IOException { + NamedAnalyzer newDefaultSearchAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchAnalyzer(), indexSettings); + NamedAnalyzer newDefaultSearchQuoteAnalyzer = rebuildIfNecessary(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), indexSettings); + Map newAnalyzers = new HashMap<>(); + for (NamedAnalyzer analyzer : indexAnalyzers.getAnalyzers().values()) { + newAnalyzers.put(analyzer.name(), rebuildIfNecessary(analyzer, indexSettings)); + } + return new IndexAnalyzers(indexSettings, indexAnalyzers.getDefaultIndexAnalyzer(), newDefaultSearchAnalyzer, + newDefaultSearchQuoteAnalyzer, newAnalyzers, indexAnalyzers.getNormalizers(), indexAnalyzers.getWhitespaceNormalizers()); + } + + /** + * Check if the input analyzer needs to be rebuilt. If not, return analyzer unaltered, otherwise rebuild it. We currently only consider + * instances of {@link CustomAnalyzer} with {@link AnalysisMode#SEARCH_TIME} to be eligible for rebuilding. + */ + private NamedAnalyzer rebuildIfNecessary(NamedAnalyzer oldAnalyzer, IndexSettings indexSettings) + throws IOException { + // only rebuild custom analyzers that are in SEARCH_TIME mode + if ((oldAnalyzer.getAnalysisMode() == AnalysisMode.SEARCH_TIME) == false + || (oldAnalyzer.analyzer() instanceof CustomAnalyzer == false)) { + return oldAnalyzer; + } else { + String analyzerName = oldAnalyzer.name(); + + // get tokenizer necessary to re-build the analyzer + String tokenizer = indexSettings.getSettings().get("index.analysis.analyzer." + analyzerName + ".tokenizer"); + Map> tokenizerProvider = Collections.singletonMap(tokenizer, + getTokenizerProvider(tokenizer, indexSettings)); + final Map tokenizerSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER); + final Map tokenizers = buildMapping(Component.TOKENIZER, indexSettings, tokenizerSettings, + tokenizerProvider, Collections.emptyMap()); + + // get char filters necessary to re-build the analyzer + List charFilterNames = indexSettings.getSettings() + .getAsList("index.analysis.analyzer." + analyzerName + ".char_filter"); + Map> charFilterProvider = charFilterNames.stream() + .map(s -> new AbstractMap.SimpleEntry<>(s, getCharFilterProvider(s, indexSettings))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + final Map charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER); + final Map charFilters = buildMapping(Component.CHAR_FILTER, indexSettings, + charFiltersSettings, charFilterProvider, Collections.emptyMap()); + + // get token filters necessary to re-build the analyzer + List tokenFilterNames = indexSettings.getSettings().getAsList("index.analysis.analyzer." + analyzerName + ".filter"); + final Map tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER); + Map> tokenFilterProviders = new HashMap<>(); + for (String filterName : tokenFilterNames) { + AnalysisProvider tokenFilterProvider = getTokenFilterProvider(filterName, indexSettings); + Settings settings = tokenFiltersSettings.get(filterName); + if (settings != null) { + String type = settings.get("type"); + if (type != null) { + AnalysisProvider provider = getTokenFilterProvider(type, indexSettings); + tokenFilterProviders.put(type, provider); + } + } else { + tokenFilterProviders.put(filterName, tokenFilterProvider); + } + } + + final Map tokenFilters = buildMapping(Component.FILTER, indexSettings, + tokenFiltersSettings, tokenFilterProviders, prebuiltAnalysis.preConfiguredTokenFilters); + + Settings analyzerSettings = indexSettings.getSettings().getAsSettings("index.analysis.analyzer." + analyzerName); + AnalyzerProvider analyzerProvider = new CustomAnalyzerProvider(indexSettings, analyzerName, analyzerSettings); + + // produce the analyzer + return AnalysisRegistry.produceAnalyzer(analyzerName, analyzerProvider, tokenFilters, charFilters, tokenizers); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java index 4cb0b9aa324c9..0ebf7360f2c3a 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java @@ -24,6 +24,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -54,9 +55,9 @@ public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAna this.defaultIndexAnalyzer = defaultIndexAnalyzer; this.defaultSearchAnalyzer = defaultSearchAnalyzer; this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer; - this.analyzers = unmodifiableMap(analyzers); - this.normalizers = unmodifiableMap(normalizers); - this.whitespaceNormalizers = unmodifiableMap(whitespaceNormalizers); + this.analyzers = unmodifiableMap(new HashMap<>(analyzers)); + this.normalizers = unmodifiableMap(new HashMap<>(normalizers)); + this.whitespaceNormalizers = unmodifiableMap(new HashMap<>(whitespaceNormalizers)); } /** @@ -66,6 +67,13 @@ public NamedAnalyzer get(String name) { return analyzers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index analyzers + */ + Map getAnalyzers() { + return analyzers; + } + /** * Returns a normalizer mapped to the given name or null if not present */ @@ -73,6 +81,13 @@ public NamedAnalyzer getNormalizer(String name) { return normalizers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index normalizers + */ + Map getNormalizers() { + return normalizers; + } + /** * Returns a normalizer that splits on whitespace mapped to the given name or null if not present */ @@ -80,6 +95,13 @@ public NamedAnalyzer getWhitespaceNormalizer(String name) { return whitespaceNormalizers.get(name); } + /** + * Returns an (unmodifiable) map of containing the index whitespace normalizers + */ + Map getWhitespaceNormalizers() { + return whitespaceNormalizers; + } + /** * Returns the default index analyzer for this index */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java index 27d061d8c2788..75ebc9eecc466 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java @@ -86,6 +86,18 @@ public FieldTypeLookup copyAndAddAll(String type, return new FieldTypeLookup(fullName, aliases); } + /** + * Return a new instance that contains the union of this instance and the mapped field types + */ + public FieldTypeLookup copyAndAddAll(Collection mappedFieldTypes) { + CopyOnWriteHashMap fullName = this.fullNameToFieldType; + CopyOnWriteHashMap aliases = this.aliasToConcreteName; + + for (MappedFieldType mft : mappedFieldTypes) { + fullName = fullName.copyAndPut(mft.name(), mft); + } + return new FieldTypeLookup(fullName, aliases); + } /** Returns the field for the given field */ public MappedFieldType get(String field) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 487a6ac4789e3..fba9220eb703a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -46,6 +46,8 @@ import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.analysis.AnalysisMode; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.Mapper.BuilderContext; @@ -70,7 +72,9 @@ import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -122,7 +126,7 @@ public enum MergeReason { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(MapperService.class)); - private final IndexAnalyzers indexAnalyzers; + private IndexAnalyzers indexAnalyzers; private volatile String defaultMappingSource; @@ -136,8 +140,8 @@ public enum MergeReason { private final DocumentMapperParser documentParser; private final MapperAnalyzerWrapper indexAnalyzer; - private final MapperAnalyzerWrapper searchAnalyzer; - private final MapperAnalyzerWrapper searchQuoteAnalyzer; + private MapperAnalyzerWrapper searchAnalyzer; + private MapperAnalyzerWrapper searchQuoteAnalyzer; private volatile Map unmappedFieldTypes = emptyMap(); @@ -843,4 +847,30 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { return defaultAnalyzer; } } + + public void reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException { + logger.info("reloading search analyzers"); + + // refresh indexAnalyzers and search analyzers + this.indexAnalyzers = registry.rebuildIndexAnalyzers(this.indexAnalyzers, indexSettings); + this.searchAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); + this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(this.indexAnalyzers.getDefaultSearchQuoteAnalyzer(), + p -> p.searchQuoteAnalyzer()); + + // also reload search time analyzers in MappedFieldTypes + // refresh search time analyzers in MappedFieldTypes + List mftsToRefresh = StreamSupport.stream(fieldTypes.spliterator(), false) + .filter(mft -> (mft.searchAnalyzer() != null && mft.searchAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME) + || (mft.searchQuoteAnalyzer() != null && mft.searchQuoteAnalyzer().getAnalysisMode() == AnalysisMode.SEARCH_TIME)) + .collect(Collectors.toList()); + List updated = mftsToRefresh.stream().map(mft -> { + MappedFieldType newMft = mft.clone(); + newMft.setSearchAnalyzer(indexAnalyzers.get(mft.searchAnalyzer().name())); + newMft.setSearchQuoteAnalyzer(indexAnalyzers.get(mft.searchQuoteAnalyzer().name())); + newMft.freeze(); + return newMft; + }).collect(Collectors.toList()); + fieldTypes = fieldTypes.copyAndAddAll(updated); + // mapper.root().updateFieldType(); + } }