Merge branch 'dev-simple_query_string-#192' into dev-simple_query_str…

…ing-#192-impl3

Signed-off-by: Yury Fridlyand <[email protected]>

build.gradle

@@ -6,9 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.0.0-rc1-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.0.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "rc1")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
         if (buildVersionQualifier) {
@@ -46,11 +46,6 @@ repositories {
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
 }
 
-ext {
-    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-    buildVersionQualifier = System.getProperty("build.version_qualifier", "rc1")
-}
-
 allprojects {
     version = opensearch_version.tokenize('-')[0] + '.0'
     if (buildVersionQualifier) {

core/src/main/java/org/opensearch/sql/expression/DSL.java

@@ -651,12 +651,18 @@ public FunctionExpression castDatetime(Expression value) {
   }
 
   public FunctionExpression match(Expression... args) {
-    return (FunctionExpression) repository
-        .compile(BuiltinFunctionName.MATCH.getName(), Arrays.asList(args.clone()));
+    return compile(BuiltinFunctionName.MATCH, args);
+  }
+
+  public FunctionExpression match_phrase(Expression... args) {
+    return compile(BuiltinFunctionName.MATCH_PHRASE, args);
   }
 
   public FunctionExpression simple_query_string(Expression... args) {
-    return (FunctionExpression) repository
-        .compile(BuiltinFunctionName.SIMPLE_QUERY_STRING.getName(), Arrays.asList(args.clone()));
+    return compile(BuiltinFunctionName.SIMPLE_QUERY_STRING, args);
+  }
+
+  private FunctionExpression compile(BuiltinFunctionName bfn, Expression... args) {
+    return (FunctionExpression) repository.compile(bfn.getName(), Arrays.asList(args.clone()));
   }
 }

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -188,6 +188,8 @@ public enum BuiltinFunctionName {
    */
   MATCH(FunctionName.of("match")),
   SIMPLE_QUERY_STRING(FunctionName.of("simple_query_string")),
+  MATCH_PHRASE(FunctionName.of("match_phrase")),
+  MATCHPHRASE(FunctionName.of("matchphrase")),
 
   /**
    * Legacy Relevance Function.

core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java

@@ -25,23 +25,37 @@
 
 @UtilityClass
 public class OpenSearchFunctions {
+
+  public static final int MATCH_MAX_NUM_PARAMETERS = 14;
+  public static final int MATCH_PHRASE_MAX_NUM_PARAMETERS = 5;
+  public static final int MIN_NUM_PARAMETERS = 2;
+  public static final int SIMPLE_QUERY_STRING_MAX_NUM_PARAMETERS = 14;
+
+  /**
+   * Add functions specific to OpenSearch to repository.
+   */
   public void register(BuiltinFunctionRepository repository) {
     repository.register(match());
     repository.register(simple_query_string());
+    // Register MATCHPHRASE as MATCH_PHRASE as well for backwards
+    // compatibility.
+    repository.register(match_phrase(BuiltinFunctionName.MATCH_PHRASE));
+    repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE));
   }
 
   private static FunctionResolver match() {
     FunctionName funcName = BuiltinFunctionName.MATCH.getName();
-    // At most field, query, and all optional parameters
-    final int matchMaxNumParameters = 14;
-    return getRelevanceFunctionResolver(funcName, matchMaxNumParameters, STRING);
+    return getRelevanceFunctionResolver(funcName, MATCH_MAX_NUM_PARAMETERS, STRING);
+  }
+
+  private static FunctionResolver match_phrase(BuiltinFunctionName matchPhrase) {
+    FunctionName funcName = matchPhrase.getName();
+    return getRelevanceFunctionResolver(funcName, MATCH_PHRASE_MAX_NUM_PARAMETERS, STRING);
   }
 
   private static FunctionResolver simple_query_string() {
     FunctionName funcName = BuiltinFunctionName.SIMPLE_QUERY_STRING.getName();
-    // At most field, query, and all optional parameters
-    final int simpleQueryStringMaxNumParameters = 12;
-    return getRelevanceFunctionResolver(funcName, simpleQueryStringMaxNumParameters, STRUCT);
+    return getRelevanceFunctionResolver(funcName, SIMPLE_QUERY_STRING_MAX_NUM_PARAMETERS, STRUCT);
   }
 
   private static FunctionResolver getRelevanceFunctionResolver(
@@ -52,10 +66,10 @@ private static FunctionResolver getRelevanceFunctionResolver(
 
   private static Map<FunctionSignature, FunctionBuilder> getRelevanceFunctionSignatureMap(
       FunctionName funcName, int maxNumParameters, ExprCoreType firstArgType) {
-    final int minNumParameters = 2;
     FunctionBuilder buildFunction = args -> new OpenSearchFunction(funcName, args);
     var signatureMapBuilder = ImmutableMap.<FunctionSignature, FunctionBuilder>builder();
-    for (int numParameters = minNumParameters; numParameters <= maxNumParameters; numParameters++) {
+    for (int numParameters = MIN_NUM_PARAMETERS;
+         numParameters <= maxNumParameters; numParameters++) {
       List<ExprType> args = new ArrayList<>(Collections.nCopies(numParameters - 1, STRING));
       args.add(0, firstArgType);
       signatureMapBuilder.put(new FunctionSignature(funcName, args), buildFunction);

core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java

@@ -11,6 +11,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import org.junit.jupiter.api.Test;
 import org.opensearch.sql.data.model.ExprTupleValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
@@ -55,6 +56,8 @@ public class OpenSearchFunctionsTest extends ExpressionTestBase {
       "zero_terms_query", DSL.literal("None"));
   private final NamedArgumentExpression boost = new NamedArgumentExpression(
       "boost", DSL.literal("2.0"));
+  private final NamedArgumentExpression slop = new NamedArgumentExpression(
+      "slop", DSL.literal("3"));
 
   @Test
   void match() {
@@ -120,6 +123,23 @@ void match() {
     assertEquals(BOOLEAN, expr.type());
   }
 
+  @Test
+  void match_phrase() {
+    for (FunctionExpression expr : match_phrase_dsl_expressions()) {
+      assertEquals(BOOLEAN, expr.type());
+    }
+  }
+
+
+  List<FunctionExpression> match_phrase_dsl_expressions() {
+    return List.of(
+      dsl.match_phrase(field, query),
+      dsl.match_phrase(field, query, analyzer),
+      dsl.match_phrase(field, query, analyzer, zeroTermsQueryAll),
+      dsl.match_phrase(field, query, analyzer, zeroTermsQueryNone, slop)
+    );
+  }
+
   @Test
   void match_in_memory() {
     FunctionExpression expr = dsl.match(field, query);

docs/user/dql/functions.rst

@@ -2183,8 +2183,6 @@ Example with only ``field`` and ``query`` expressions, and all other parameters
     | Bates      | 789 Madison Street |
     +------------+--------------------+
 
-
-
 Another example to show how to set custom values for the optional parameters::
 
     os> SELECT lastname FROM accounts WHERE match(firstname, 'Hattie', operator='AND', boost=2.0);
@@ -2196,6 +2194,45 @@ Another example to show how to set custom values for the optional parameters::
     +------------+
 
 
+MATCH_PHRASE
+------------
+
+Description
+>>>>>>>>>>>
+
+``match_phrase(field_expression, query_expression[, option=<option_value>]*)``
+
+The match_phrase function maps to the match_phrase query used in search engine, to return the documents that match a provided text with a given field. Available parameters include:
+
+- analyzer
+- slop
+- zero_terms_query
+
+For backward compatibility, matchphrase is also supported and mapped to match_phrase query as well.
+
+Example with only ``field`` and ``query`` expressions, and all other parameters are set default values::
+
+    os> SELECT author, title FROM books WHERE match_phrase(author, 'Alexander Milne');
+    fetched rows / total rows = 2/2
+    +----------------------+--------------------------+
+    | author               | title                    |
+    |----------------------+--------------------------|
+    | Alan Alexander Milne | The House at Pooh Corner |
+    | Alan Alexander Milne | Winnie-the-Pooh          |
+    +----------------------+--------------------------+
+
+Another example to show how to set custom values for the optional parameters::
+
+    os> SELECT author, title FROM books WHERE match_phrase(author, 'Alan Milne', slop = 2);
+    fetched rows / total rows = 2/2
+    +----------------------+--------------------------+
+    | author               | title                    |
+    |----------------------+--------------------------|
+    | Alan Alexander Milne | The House at Pooh Corner |
+    | Alan Alexander Milne | Winnie-the-Pooh          |
+    +----------------------+--------------------------+
+
+
 SIMPLE_QUERY_STRING
 -------------------
 
@@ -2237,8 +2274,6 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters
     | Nanette     | Bates      | Nogal  | 789 Madison Street |
     +-------------+------------+--------+--------------------+
 
-
-
 Another example to show how to set custom values for the optional parameters::
 
     os> select firstname, lastname, city, address from accounts where simple_query_string(['firstname', city ^ 2], 'Amber Nogal', analyzer=keyword, default_operator='AND');

docs/user/ppl/functions/relevance.rst

@@ -57,6 +57,47 @@ Another example to show how to set custom values for the optional parameters::
     +------------+
 
 
+MATCH_PHRASE
+------------
+
+Description
+>>>>>>>>>>>
+
+``match_phrase(field_expression, query_expression[, option=<option_value>]*)``
+
+The match_phrase function maps to the match_phrase query used in search engine, to return the documents that match a provided text with a given field. Available parameters include:
+
+- analyzer
+- slop
+- zero_terms_query
+
+For backward compatibility, matchphrase is also supported and mapped to match_phrase query as well.
+
+Example with only ``field`` and ``query`` expressions, and all other parameters are set default values::
+
+    os> source=books | where match_phrase(author, 'Alexander Milne') | fields author, title
+    fetched rows / total rows = 2/2
+    +----------------------+--------------------------+
+    | author               | title                    |
+    |----------------------+--------------------------|
+    | Alan Alexander Milne | The House at Pooh Corner |
+    | Alan Alexander Milne | Winnie-the-Pooh          |
+    +----------------------+--------------------------+
+
+
+
+Another example to show how to set custom values for the optional parameters::
+
+    os> source=books | where match_phrase(author, 'Alan Milne', slop = 2) | fields author, title
+    fetched rows / total rows = 2/2
+    +----------------------+--------------------------+
+    | author               | title                    |
+    |----------------------+--------------------------|
+    | Alan Alexander Milne | The House at Pooh Corner |
+    | Alan Alexander Milne | Winnie-the-Pooh          |
+    +----------------------+--------------------------+
+
+
 SIMPLE_QUERY_STRING
 -------------------
 
@@ -99,8 +140,6 @@ Example with only ``fields`` and ``query`` expressions, and all other parameters
     | Nanette     | Bates      | Nogal  | 789 Madison Street |
     +-------------+------------+--------+--------------------+
 
-
-
 Another example to show how to set custom values for the optional parameters::
 
     os> source=accounts | where simple_query_string(['firstname', city ^ 2], 'Amber Nogal', analyzer=keyword, default_operator='AND') | fields firstname, lastname, city, address;
@@ -110,6 +149,7 @@ Another example to show how to set custom values for the optional parameters::
     |-------------+------------+--------+-----------|
     +-------------+------------+--------+-----------+
 
+
 Limitations
 >>>>>>>>>>>
 

doctest/build.gradle

@@ -49,10 +49,14 @@ build.dependsOn doctest
 clean.dependsOn(cleanBootstrap)
 
 // 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT
-String opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
-String opensearch_build = opensearch_no_snapshot.tokenize('-')[0] + '.0-' + opensearch_no_snapshot.tokenize('-')[1]
-String mlCommonsRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/2500/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-' + opensearch_build +'.zip'
-String mlCommonsPlugin = "ml-commons"
+String opensearch_no_snapshot = opensearch_version.replace('-SNAPSHOT', '')
+String[] version_tokens = opensearch_no_snapshot.tokenize('-')
+String opensearch_build = version_tokens[0] + '.0'
+if (version_tokens.length > 1) {
+    opensearch_build += '-' + version_tokens[1]
+}
+String mlCommonsRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-' + opensearch_build + '.zip'
+String mlCommonsPlugin = 'opensearch-ml'
 
 testClusters {
     docTestCluster {
@@ -66,9 +70,9 @@ testClusters {
                         if (!dir.exists()) {
                             dir.mkdirs()
                         }
-                        File f = new File(dir, mlCommonsPlugin)
+                        File f = new File(dir, mlCommonsPlugin + '-' + opensearch_build + '.zip')
                         if (!f.exists()) {
-                            new URL(mlCommonsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
+                            new URL(mlCommonsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins } }
                         }
                         return fileTree(mlCommonsPlugin).getSingleFile()
                     }

doctest/opensearch-ml/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

doctest/test_data/books.json

@@ -0,0 +1,2 @@
+{"id": 1, "author": "Alan Alexander Milne", "title": "The House at Pooh Corner"}
+{"id": 2, "author": "Alan Alexander Milne", "title": "Winnie-the-Pooh"}

doctest/test_docs.py

@@ -24,6 +24,7 @@
 PEOPLE = "people"
 ACCOUNT2 = "account2"
 NYC_TAXI = "nyc_taxi"
+BOOKS = "books"
 
 
 class DocTestConnection(OpenSearchConnection):
@@ -88,6 +89,7 @@ def set_up_test_indices(test):
     load_file("people.json", index_name=PEOPLE)
     load_file("account2.json", index_name=ACCOUNT2)
     load_file("nyc_taxi.json", index_name=NYC_TAXI)
+    load_file("books.json", index_name=BOOKS)
 
 
 def load_file(filename, index_name):
@@ -116,7 +118,7 @@ def set_up(test):
 
 def tear_down(test):
     # drop leftover tables after each test
-    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI], ignore_unavailable=True)
+    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS], ignore_unavailable=True)
 
 
 docsuite = partial(doctest.DocFileSuite,

integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestConfigTest.java

@@ -11,7 +11,7 @@
 import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;

integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestDataSetTest.java

@@ -8,7 +8,7 @@
 
 import static org.hamcrest.Matchers.contains;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import org.junit.Test;
 import org.opensearch.sql.correctness.testset.TestDataSet;

integ-test/src/test/java/org/opensearch/sql/correctness/tests/TestQuerySetTest.java

@@ -7,7 +7,7 @@
 package org.opensearch.sql.correctness.tests;
 
 import static org.hamcrest.Matchers.contains;
-import static org.junit.Assert.assertThat;
+import static org.hamcrest.MatcherAssert.assertThat;
 
 import org.junit.Test;
 import org.opensearch.sql.correctness.testset.TestQuerySet;

integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java

@@ -255,8 +255,4 @@ public void aggregateCastStatementShouldNotReturnZero() {
     verifySchema(response, schema("SUM(CAST(male AS INT))", "male_sum", "integer"));
     verifyDataRows(response, rows(4));
   }
-
-  private JSONObject executeJdbcRequest(String query) {
-    return new JSONObject(executeQuery(query, "jdbc"));
-  }
 }

integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java

@@ -1355,8 +1355,4 @@ private double getDoubleAggregationValue(final JSONObject queryResult,
 
     return targetField.getDouble(subFieldName);
   }
-
-  private JSONObject executeJdbcRequest(String query) {
-    return new JSONObject(executeQuery(query, "jdbc"));
-  }
 }

integ-test/src/test/java/org/opensearch/sql/legacy/DateFormatIT.java

@@ -262,8 +262,4 @@ public static String getScriptAggregationKey(JSONObject aggregation, String pref
         .orElseThrow(() -> new RuntimeException(
             "Can't find key" + prefix + " in aggregation " + aggregation));
   }
-
-  private JSONObject executeJdbcRequest(String query) {
-    return new JSONObject(executeQuery(query, "jdbc"));
-  }
 }