From 595ab3224b3edc4a555b62c87fa0e8838671360c Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Mon, 21 Nov 2022 08:12:02 -0800 Subject: [PATCH 1/3] Added Tests Signed-off-by: GabeFernandez310 --- .../org/opensearch/sql/sql/MatchPhraseIT.java | 19 ++ .../filter/lucene/MatchPhraseQueryTest.java | 172 +++++++++++++++++- .../sql/sql/antlr/SQLSyntaxParserTest.java | 17 ++ .../sql/parser/AstExpressionBuilderTest.java | 16 ++ 4 files changed, 222 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java index 0773238948..b870a60604 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java @@ -36,10 +36,29 @@ public void test_matchphrase_legacy_function() throws IOException { verifyDataRows(result, rows("quick fox"), rows("quick fox here")); } + @Test + public void test_matchphrasequery_legacy_function() throws IOException { + String query = "SELECT phrase FROM %s WHERE matchphrasequery(phrase, 'quick fox')"; + JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); + verifyDataRows(result, rows("quick fox"), rows("quick fox here")); + } + @Test public void test_match_phrase_with_slop() throws IOException { String query = "SELECT phrase FROM %s WHERE match_phrase(phrase, 'brown fox', slop = 2)"; JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); verifyDataRows(result, rows("brown fox"), rows("fox brown")); } + + @Test + public void test_alternate_syntax_for_match_phrase_returns_same_result() throws IOException { + String query1 = "SELECT phrase FROM %s WHERE matchphrase(phrase, 'quick fox')"; + String query2 = "SELECT phrase FROM %s WHERE match_phrase(phrase, 'quick fox')"; + String query3 = "SELECT phrase FROM %s WHERE matchphrasequery(phrase, 'quick fox')"; + JSONObject result1 = executeJdbcRequest(String.format(query1, TEST_INDEX_PHRASE)); + JSONObject result2 = executeJdbcRequest(String.format(query2, TEST_INDEX_PHRASE)); + JSONObject result3 = executeJdbcRequest(String.format(query3, TEST_INDEX_PHRASE)); + assertTrue(result1.similar(result2)); + assertTrue(result1.similar(result3)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java index 78b000bba3..6a298326b7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java @@ -28,7 +28,9 @@ public class MatchPhraseQueryTest { private final MatchPhraseQuery matchPhraseQuery = new MatchPhraseQuery(); - private final FunctionName matchPhrase = FunctionName.of("match_phrase"); + private final FunctionName matchPhraseName = FunctionName.of("matchphrase"); + private final FunctionName matchPhraseWithUnderscoreName = FunctionName.of("match_phrase"); + private final FunctionName matchPhraseQueryName = FunctionName.of("matchphrasequery"); @Test public void test_SyntaxCheckException_when_no_arguments() { @@ -102,9 +104,175 @@ public void test_zero_terms_query_parameter_lower_case() { Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } + @Test + public void test_SyntaxCheckException_when_no_arguments_match_phrase_syntax() { + List arguments = List.of(); + assertThrows(SyntaxCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_SyntaxCheckException_when_one_argument_match_phrase_syntax() { + List arguments = List.of(DSL.namedArgument("field", "test")); + assertThrows(SyntaxCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + + } + + @Test + public void test_SyntaxCheckException_when_invalid_parameter_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "test"), + DSL.namedArgument("query", "test2"), + DSL.namedArgument("unsupported", "3")); + Assertions.assertThrows(SemanticCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_analyzer_parameter_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("analyzer", "standard") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void build_succeeds_with_two_arguments_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "test"), + DSL.namedArgument("query", "test2")); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_slop_parameter_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("slop", "2") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_zero_terms_query_parameter_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("zero_terms_query", "ALL") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_zero_terms_query_parameter_lower_case_match_phrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("zero_terms_query", "all") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseWithUnderscoreName))); + } + + @Test + public void test_SyntaxCheckException_when_no_arguments_matchphrase_syntax() { + List arguments = List.of(); + assertThrows(SyntaxCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void test_SyntaxCheckException_when_one_argument_matchphrase_syntax() { + List arguments = List.of(DSL.namedArgument("field", "test")); + assertThrows(SyntaxCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + + } + + @Test + public void test_SyntaxCheckException_when_invalid_parameter_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "test"), + DSL.namedArgument("query", "test2"), + DSL.namedArgument("unsupported", "3")); + Assertions.assertThrows(SemanticCheckException.class, + () -> matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void test_analyzer_parameter_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("analyzer", "standard") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void build_succeeds_with_two_arguments_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "test"), + DSL.namedArgument("query", "test2")); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void test_slop_parameter_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("slop", "2") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void test_zero_terms_query_parameter_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("zero_terms_query", "ALL") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + + @Test + public void test_zero_terms_query_parameter_lower_case_matchphrase_syntax() { + List arguments = List.of( + DSL.namedArgument("field", "t1"), + DSL.namedArgument("query", "t2"), + DSL.namedArgument("zero_terms_query", "all") + ); + Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( + arguments, matchPhraseQueryName))); + } + private class MatchPhraseExpression extends FunctionExpression { public MatchPhraseExpression(List arguments) { - super(MatchPhraseQueryTest.this.matchPhrase, arguments); + super(matchPhraseName, arguments); + } + + public MatchPhraseExpression(List arguments, FunctionName funcName) { + super(funcName, arguments); } @Override diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 6b78376d45..1d62278b73 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -391,6 +391,7 @@ public void can_parse_match_relevance_function() { "matchPhraseComplexQueries", "matchPhraseGeneratedQueries", "generateMatchPhraseQueries", + "matchPhraseQueryComplexQueries" }) public void canParseComplexMatchPhraseArgsTest(String query) { assertNotNull(parser.parse(query)); @@ -420,6 +421,22 @@ private static Stream matchPhraseComplexQueries() { ); } + private static Stream matchPhraseQueryComplexQueries() { + return Stream.of( + "SELECT * FROM t WHERE matchphrasequery(c, 3)", + "SELECT * FROM t WHERE matchphrasequery(c, 3, fuzziness=AUTO)", + "SELECT * FROM t WHERE matchphrasequery(c, 3, zero_terms_query=\"all\")", + "SELECT * FROM t WHERE matchphrasequery(c, 3, lenient=true)", + "SELECT * FROM t WHERE matchphrasequery(c, 3, lenient='true')", + "SELECT * FROM t WHERE matchphrasequery(c, 3, operator=xor)", + "SELECT * FROM t WHERE matchphrasequery(c, 3, cutoff_frequency=0.04)", + "SELECT * FROM t WHERE matchphrasequery(c, 3, cutoff_frequency=0.04, analyzer = english, " + + "prefix_length=34, fuzziness='auto', minimum_should_match='2<-25% 9<-3')", + "SELECT * FROM t WHERE matchphrasequery(c, 3, minimum_should_match='2<-25% 9<-3')", + "SELECT * FROM t WHERE matchphrasequery(c, 3, operator='AUTO')" + ); + } + private static Stream matchPhraseGeneratedQueries() { var matchArgs = new HashMap(); matchArgs.put("fuzziness", new String[]{ "AUTO", "AUTO:1,5", "1" }); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index cb00ea2f18..869ae2b8ab 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -451,6 +451,22 @@ public void filteredDistinctCount() { ); } + @Test + public void matchPhraseQueryAllParameters() { + assertEquals( + AstDSL.function("matchphrasequery", + unresolvedArg("field", stringLiteral("test")), + unresolvedArg("query", stringLiteral("search query")), + unresolvedArg("slop", stringLiteral("3")), + unresolvedArg("analyzer", stringLiteral("standard")), + unresolvedArg("zero_terms_query", stringLiteral("NONE")) + ), + buildExprAst("matchphrasequery(test, 'search query', slop = 3" + + ", analyzer = 'standard', zero_terms_query='NONE'" + + ")") + ); + } + @Test public void matchPhrasePrefixAllParameters() { assertEquals( From 5c48bb5c49ec2631851fa2528cc0a18b6daea21d Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Mon, 21 Nov 2022 08:13:35 -0800 Subject: [PATCH 2/3] Changed Documentation Signed-off-by: GabeFernandez310 --- docs/user/dql/functions.rst | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 788cac0433..035ef764cb 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -2769,6 +2769,46 @@ Another example to show how to set custom values for the optional parameters:: +----------------------+--------------------------+ +MATCHPHRASEQUERY +------------ + +Description +>>>>>>>>>>> + +``matchphrasequery(field_expression, query_expression[, option=]*)`` + +The matchphrasequery function maps to the match_phrase query used in search engine, to return the documents that match a provided text with a given field. +It is an alternate syntax for the `match_phrase`_ function. Available parameters include: + +- analyzer +- slop +- zero_terms_query + +For backward compatibility, matchphrase is also supported and mapped to match_phrase query. + +Example with only ``field`` and ``query`` expressions, and all other parameters are set default values:: + + os> SELECT author, title FROM books WHERE match_phrase(author, 'Alexander Milne'); + fetched rows / total rows = 2/2 + +----------------------+--------------------------+ + | author | title | + |----------------------+--------------------------| + | Alan Alexander Milne | The House at Pooh Corner | + | Alan Alexander Milne | Winnie-the-Pooh | + +----------------------+--------------------------+ + +Another example to show how to set custom values for the optional parameters:: + + os> SELECT author, title FROM books WHERE match_phrase(author, 'Alan Milne', slop = 2); + fetched rows / total rows = 2/2 + +----------------------+--------------------------+ + | author | title | + |----------------------+--------------------------| + | Alan Alexander Milne | The House at Pooh Corner | + | Alan Alexander Milne | Winnie-the-Pooh | + +----------------------+--------------------------+ + + MATCH_BOOL_PREFIX ----- From fbcee86164e1ec89580bb2392c15d816f0ff452b Mon Sep 17 00:00:00 2001 From: GabeFernandez310 Date: Mon, 21 Nov 2022 08:13:49 -0800 Subject: [PATCH 3/3] Added Implementation. Signed-off-by: GabeFernandez310 --- .../opensearch/sql/expression/function/BuiltinFunctionName.java | 1 + .../opensearch/sql/expression/function/OpenSearchFunctions.java | 1 + .../opensearch/storage/script/filter/FilterQueryBuilder.java | 1 + sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 2 +- 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 51d91eb372..af18d28b18 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -210,6 +210,7 @@ public enum BuiltinFunctionName { SIMPLE_QUERY_STRING(FunctionName.of("simple_query_string")), MATCH_PHRASE(FunctionName.of("match_phrase")), MATCHPHRASE(FunctionName.of("matchphrase")), + MATCHPHRASEQUERY(FunctionName.of("matchphrasequery")), QUERY_STRING(FunctionName.of("query_string")), MATCH_BOOL_PREFIX(FunctionName.of("match_bool_prefix")), HIGHLIGHT(FunctionName.of("highlight")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index 97afe3675e..3575ebb4e2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -36,6 +36,7 @@ public void register(BuiltinFunctionRepository repository) { // compatibility. repository.register(match_phrase(BuiltinFunctionName.MATCH_PHRASE)); repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE)); + repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASEQUERY)); repository.register(match_phrase_prefix()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java index ab8fb562da..843096eee0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java @@ -61,6 +61,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor