From 4edcd30bf1d884e3ddd8215a4102c128deb4d31c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 24 Oct 2018 09:37:10 +0100 Subject: [PATCH 1/5] WIP --- .../index/analysis/IcuAnalyzerProvider.java | 47 +++++++++++++++ .../analysis/icu/AnalysisICUPlugin.java | 8 +++ .../index/analysis/IcuAnalyzerTest.java | 60 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java create mode 100644 plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java new file mode 100644 index 0000000000000..b58ebed01bd2e --- /dev/null +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java @@ -0,0 +1,47 @@ +package org.elasticsearch.index.analysis; + +import com.ibm.icu.text.Normalizer2; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.icu.ICUFoldingFilter; +import org.apache.lucene.analysis.icu.ICUNormalizer2CharFilter; +import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; + +import java.io.Reader; + +public class IcuAnalyzerProvider extends AbstractIndexAnalyzerProvider { + + private final Normalizer2 normalizer; + + public IcuAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + super(indexSettings, name, settings); + String method = settings.get("method", "nfkc_cf"); + String mode = settings.get("mode"); + if (!"compose".equals(mode) && !"decompose".equals(mode)) { + mode = "compose"; + } + Normalizer2 normalizer = Normalizer2.getInstance( + null, method, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE); + this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(normalizer, settings); + } + + @Override + public Analyzer get() { + return new Analyzer() { + + @Override + protected Reader initReader(String fieldName, Reader reader) { + return new ICUNormalizer2CharFilter(reader, normalizer); + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new ICUTokenizer(); + return new TokenStreamComponents(source, new ICUFoldingFilter(source)); + } + }; + } +} diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java index 58ebdc8e2a801..9d5ed5af7bdca 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java @@ -21,8 +21,11 @@ import static java.util.Collections.singletonMap; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; +import org.elasticsearch.index.analysis.IcuAnalyzerProvider; import org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory; import org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory; import org.elasticsearch.index.analysis.IcuNormalizerCharFilterFactory; @@ -60,6 +63,11 @@ public Map> getTokenFilters() { return extra; } + @Override + public Map>> getAnalyzers() { + return singletonMap("icu_analyzer", IcuAnalyzerProvider::new); + } + @Override public Map> getTokenizers() { return singletonMap("icu_tokenizer", IcuTokenizerFactory::new); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java new file mode 100644 index 0000000000000..e36e8f30e623e --- /dev/null +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java @@ -0,0 +1,60 @@ +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; + +public class IcuAnalyzerTest extends BaseTokenStreamTestCase { + + public void testMixedAlphabetTokenization() throws IOException { + + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); + + String input = "안녕은하철도999극장판2.1981년8월8일.일본개봉작1999년재더빙video판"; + + Analyzer analyzer = new IcuAnalyzerProvider(idxSettings, null, "icu", settings).get(); + assertAnalyzesTo(analyzer, input, + new String[]{"안녕은하철도", "999", "극장판", "2.1981", "년", "8", "월", "8", "일", "일본개봉작", "1999", "년재더빙", "video", "판"}); + + } + + public void testMiddleDots() throws IOException { + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); + + String input = "경승지·산악·협곡·해협·곶·심연·폭포·호수·급류"; + + Analyzer analyzer = new IcuAnalyzerProvider(idxSettings, null, "icu", settings).get(); + assertAnalyzesTo(analyzer, input, + new String[]{"경승지", "산악", "협곡", "해협", "곶", "심연", "폭포", "호수", "급류"}); + } + + public void testUnicodeNumericCharacters() throws IOException { + + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); + + String input = "① ② ③ ⑴ ⑵ ⑶ ¼ ⅓ ⅜ ¹ ² ³ ₁ ₂ ₃"; + + Analyzer analyzer = new IcuAnalyzerProvider(idxSettings, null, "icu", settings).get(); + assertAnalyzesTo(analyzer, input, + new String[]{"1", "2", "3", "1", "2", "3", "1/4", "1/3", "3/8", "1", "2", "3", "1", "2", "3"}); + } + + public void testZeroWidthNonJoiners() throws IOException { + + } +} From 17ca1fba92e5f95361cdc41e860300ebf5f3e76c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 29 Oct 2018 10:51:17 +0000 Subject: [PATCH 2/5] Add docs --- docs/plugins/analysis-icu.asciidoc | 18 ++++++++++++++++++ .../analysis/analyzers/lang-analyzer.asciidoc | 3 +++ .../index/analysis/IcuAnalyzerTest.java | 7 +++---- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index ef5744f7dff18..80939496931af 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -26,6 +26,24 @@ characters. :plugin_name: analysis-icu include::install_remove.asciidoc[] +[[analysis-icu-analyzer]] +==== ICU Analyzer + +Performs basic normalization, tokenization and character folding, using the +`icu_normalizer` char filter, `icu_tokenizer` and `icu_normalizer` token filter + +The following parameters are accepted: + +[horizontal] + +`method`:: + + Normalization method. Accepts `nfkc`, `nfc` or `nfkc_cf` (default) + +`mode`:: + + Normalization mode. Accepts `compose` (default) or `decompose`. + [[analysis-icu-normalization-charfilter]] ==== ICU Normalization Character Filter diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc index 78f7607b1e443..e8713123b2ca9 100644 --- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc @@ -421,6 +421,9 @@ PUT /catalan_example [[cjk-analyzer]] ===== `cjk` analyzer +NOTE: You may find that `icu_analyzer` in the ICU analysis plugin works better +for CJK text than the `cjk` analyzer. + The `cjk` analyzer could be reimplemented as a `custom` analyzer as follows: [source,js] diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java index e36e8f30e623e..6cffb21d04828 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java @@ -6,6 +6,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; @@ -21,7 +22,8 @@ public void testMixedAlphabetTokenization() throws IOException { String input = "안녕은하철도999극장판2.1981년8월8일.일본개봉작1999년재더빙video판"; - Analyzer analyzer = new IcuAnalyzerProvider(idxSettings, null, "icu", settings).get(); + AnalysisICUPlugin plugin = new AnalysisICUPlugin(); + Analyzer analyzer = plugin.getAnalyzers().get("icu_analyzer").get(idxSettings, null, "icu", settings).get(); assertAnalyzesTo(analyzer, input, new String[]{"안녕은하철도", "999", "극장판", "2.1981", "년", "8", "월", "8", "일", "일본개봉작", "1999", "년재더빙", "video", "판"}); @@ -54,7 +56,4 @@ public void testUnicodeNumericCharacters() throws IOException { new String[]{"1", "2", "3", "1", "2", "3", "1/4", "1/3", "3/8", "1", "2", "3", "1", "2", "3"}); } - public void testZeroWidthNonJoiners() throws IOException { - - } } From 1931fcaa1df2b30f08eadbf18d368aeb5784eeff Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 29 Oct 2018 11:30:06 +0000 Subject: [PATCH 3/5] precommit --- .../index/analysis/IcuAnalyzerProvider.java | 19 +++++++++++++++++ ...nalyzerTest.java => IcuAnalyzerTests.java} | 21 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) rename plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/{IcuAnalyzerTest.java => IcuAnalyzerTests.java} (75%) diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java index b58ebed01bd2e..17b773b900a35 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.index.analysis; import com.ibm.icu.text.Normalizer2; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java similarity index 75% rename from plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java rename to plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java index 6cffb21d04828..db1ad90c12333 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTest.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Analyzer; @@ -11,7 +30,7 @@ import java.io.IOException; -public class IcuAnalyzerTest extends BaseTokenStreamTestCase { +public class IcuAnalyzerTests extends BaseTokenStreamTestCase { public void testMixedAlphabetTokenization() throws IOException { From c8f49fe71ab536583a44dfefb61eb527c3227bad Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 1 Nov 2018 11:11:24 +0000 Subject: [PATCH 4/5] feedback --- .../analysis/analyzers/lang-analyzer.asciidoc | 2 +- .../index/analysis/IcuAnalyzerProvider.java | 6 +++--- .../index/analysis/IcuAnalyzerTests.java | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc index e8713123b2ca9..9a4dcbe8aaac7 100644 --- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc @@ -422,7 +422,7 @@ PUT /catalan_example ===== `cjk` analyzer NOTE: You may find that `icu_analyzer` in the ICU analysis plugin works better -for CJK text than the `cjk` analyzer. +for CJK text than the `cjk` analyzer. Experiment with your text and queries. The `cjk` analyzer could be reimplemented as a `custom` analyzer as follows: diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java index 17b773b900a35..f3a2e30af9cd4 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java @@ -38,13 +38,13 @@ public class IcuAnalyzerProvider extends AbstractIndexAnalyzerProvider public IcuAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); String method = settings.get("method", "nfkc_cf"); - String mode = settings.get("mode"); + String mode = settings.get("mode", "compose"); if (!"compose".equals(mode) && !"decompose".equals(mode)) { - mode = "compose"; + throw new IllegalArgumentException("Unknown mode [" + mode + "] in analyzer [" + name + "], expected one of [compose, decompose]"); } Normalizer2 normalizer = Normalizer2.getInstance( null, method, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE); - this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(normalizer, settings); + this.normalizer = IcuNormalizerTokenFilterFactory.wrapWithUnicodeSetFilter(indexSettings, normalizer, settings); } @Override diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java index db1ad90c12333..d15c9524db18d 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuAnalyzerTests.java @@ -30,6 +30,8 @@ import java.io.IOException; +import static org.hamcrest.Matchers.containsString; + public class IcuAnalyzerTests extends BaseTokenStreamTestCase { public void testMixedAlphabetTokenization() throws IOException { @@ -75,4 +77,20 @@ public void testUnicodeNumericCharacters() throws IOException { new String[]{"1", "2", "3", "1", "2", "3", "1/4", "1/3", "3/8", "1", "2", "3", "1", "2", "3"}); } + public void testBadSettings() { + + Settings settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("mode", "wrong") + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { + new IcuAnalyzerProvider(idxSettings, null, "icu", settings); + }); + + assertThat(e.getMessage(), containsString("Unknown mode [wrong] in analyzer [icu], expected one of [compose, decompose]")); + + } + } From 4f2fd4db292d132a543d3a84ebf8d61408384a98 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 2 Nov 2018 15:09:59 +0000 Subject: [PATCH 5/5] checkstyle --- .../org/elasticsearch/index/analysis/IcuAnalyzerProvider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java index f3a2e30af9cd4..c051d32bd4a62 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuAnalyzerProvider.java @@ -40,7 +40,8 @@ public IcuAnalyzerProvider(IndexSettings indexSettings, Environment environment, String method = settings.get("method", "nfkc_cf"); String mode = settings.get("mode", "compose"); if (!"compose".equals(mode) && !"decompose".equals(mode)) { - throw new IllegalArgumentException("Unknown mode [" + mode + "] in analyzer [" + name + "], expected one of [compose, decompose]"); + throw new IllegalArgumentException("Unknown mode [" + mode + "] in analyzer [" + name + + "], expected one of [compose, decompose]"); } Normalizer2 normalizer = Normalizer2.getInstance( null, method, "compose".equals(mode) ? Normalizer2.Mode.COMPOSE : Normalizer2.Mode.DECOMPOSE);