From 697c693ee7bb4f6ff03415603f0c24a42c9d58e8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 2 Oct 2019 11:19:25 +0100 Subject: [PATCH] Reset Token position on reuse in scripted analysis (#47424) Most of the information in AnalysisPredicateScript.Token is pulled directly from its underlying AttributeSource, but we also keep track of the token position, and this state is held directly on the Token. This information needs to be reset when the containing ScriptFilteringTokenFilter or ScriptedConditionTokenFilter is re-used. Fixes #47197 --- .../analysis/common/AnalysisPredicateScript.java | 4 ++++ .../common/PredicateTokenFilterScriptFactory.java | 6 ++++++ .../common/ScriptedConditionTokenFilterFactory.java | 7 +++++++ .../common/PredicateTokenScriptFilterTests.java | 10 +++++----- .../common/ScriptedConditionTokenFilterTests.java | 8 ++++---- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java index 3bda6f393bfdf..5d8c491efc585 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java @@ -61,6 +61,10 @@ public Token(AttributeSource source) { this.keywordAtt = source.addAttribute(KeywordAttribute.class); } + public void reset() { + this.pos = -1; + } + public void updatePosition() { this.pos = this.pos + posIncAtt.getPositionIncrement(); } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java index 84f4bb487060c..0e8a2a489fb6e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java @@ -69,5 +69,11 @@ protected boolean accept() throws IOException { token.updatePosition(); return script.execute(token); } + + @Override + public void reset() throws IOException { + super.reset(); + this.token.reset(); + } } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java index b194a5ac1114d..4e6b9ef110634 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java @@ -31,6 +31,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -119,6 +120,12 @@ protected boolean shouldFilter() { token.updatePosition(); return script.execute(token); } + + @Override + public void reset() throws IOException { + super.reset(); + token.reset(); + } } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java index 18afbdcecb3e6..84ba5e5d3373c 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java @@ -46,7 +46,7 @@ public void testSimpleFilter() throws IOException { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put("index.analysis.filter.f.type", "predicate_token_filter") - .put("index.analysis.filter.f.script.source", "token.getTerm().length() > 5") + .put("index.analysis.filter.f.script.source", "my_script") .put("index.analysis.analyzer.myAnalyzer.type", "custom") .put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard") .putList("index.analysis.analyzer.myAnalyzer.filter", "f") @@ -56,7 +56,7 @@ public void testSimpleFilter() throws IOException { AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() { @Override public boolean execute(Token token) { - return token.getTerm().length() > 5; + return token.getPosition() < 2 || token.getPosition() > 4; } }; @@ -65,7 +65,7 @@ public boolean execute(Token token) { @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); - assertEquals(new Script("token.getTerm().length() > 5"), script); + assertEquals(new Script("my_script"), script); return (FactoryType) factory; } }; @@ -79,8 +79,8 @@ public FactoryType compile(Script script, ScriptContext 5") + .put("index.analysis.filter.cond.script.source", "token.getPosition() > 1") .putList("index.analysis.filter.cond.filter", "uppercase") .put("index.analysis.analyzer.myAnalyzer.type", "custom") .put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard") @@ -56,7 +56,7 @@ public void testSimpleCondition() throws Exception { AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() { @Override public boolean execute(Token token) { - return token.getTerm().length() > 5; + return token.getPosition() > 1; } }; @@ -65,7 +65,7 @@ public boolean execute(Token token) { @Override public FactoryType compile(Script script, ScriptContext context) { assertEquals(context, AnalysisPredicateScript.CONTEXT); - assertEquals(new Script("token.getTerm().length() > 5"), script); + assertEquals(new Script("token.getPosition() > 1"), script); return (FactoryType) factory; } }; @@ -80,7 +80,7 @@ public FactoryType compile(Script script, ScriptContext