Merge remote-tracking branch 'upstream/main' into dev-datetime-now

Signed-off-by: Yury Fridlyand <[email protected]>

.github/workflows/sql-test-and-build-workflow.yml

@@ -22,9 +22,6 @@ jobs:
     - name: Build with Gradle
       run: ./gradlew build assemble
 
-    - name: Run backward compatibility tests
-      run: ./scripts/bwctest.sh
-
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds

bi-connectors/PowerBIConnector/power_bi_support.md

@@ -4,11 +4,11 @@
 * Microsoft Power BI Desktop
 * [OpenSearch](https://docs-beta.opensearch.org/opensearch/install/index/)
 * [OpenSearch SQL ODBC driver](https://docs-beta.opensearch.org/search-plugins/sql/odbc/)
-* [OpenSearch.mez](../../../bi-connectors/PowerBIConnector/bin/Release)
-* Optional: [sqlodbc_import.pbids](../../../bi-connectors/PowerBIConnector/PBIDSExamples) to help with repeated connections to the same server 
+* [OpenSearchProject.mez](OpenSearchProject.mez) or [AmazonOpenSearchService.mez](AmazonOpenSearchService.mez)
+* Optional: [sqlodbc_import.pbids](PBIDSExamples/sqlodbc_import.pbids) to help with repeated connections to the same server 
 
 ## Setup
-* Copy `OpenSearch.mez` file in the `<User>\Documents\Power BI Desktop\Custom Connectors\` folder. This will let Power BI access custom connector.
+* Copy `mez` file in the `<User>\Documents\Power BI Desktop\Custom Connectors\` folder. This will let Power BI access custom connector.
 * Open Power BI Desktop.
 * Change the security settings. Click on **Files** > **Options and settings** > **Options** > **Security** > Select **Allow any extension to load without validation or warning** for Data Extensions. This will allow the custom connector to load data into Power BI.
 
@@ -56,7 +56,7 @@
 
 More info: https://docs.microsoft.com/en-us/power-bi/connect-data/desktop-data-sources#using-pbids-files-to-get-data
 
-Example PBIDS file for OpenSearch: (available here: [sqlodbc_import.pbids](../../../bi-connectors/PowerBIConnector/PBIDSExamples/sqlodbc_import.pbids))
+Example PBIDS file for OpenSearch: (available here: [sqlodbc_import.pbids](PBIDSExamples/sqlodbc_import.pbids))
 ```json
 {
     "version": "0.1",

build.gradle

@@ -148,3 +148,22 @@ configurations.all {
     resolutionStrategy.force 'commons-codec:commons-codec:1.13'
     resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre'
 }
+
+// updateVersion: Task to auto increment to the next development iteration
+task updateVersion {
+    onlyIf { System.getProperty('newVersion') }
+    doLast {
+        ext.newVersion = System.getProperty('newVersion')
+        println "Setting version to ${newVersion}."
+        // String tokenization to support -SNAPSHOT
+        ant.replaceregexp(file:'.github/workflows/sql-workbench-test-and-build-workflow.yml', match:'OPENSEARCH_PLUGIN_VERSION: \\d+.\\d+.\\d+.\\d+', replace:'OPENSEARCH_PLUGIN_VERSION: ' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
+        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
+        ant.replaceregexp(match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true) {
+            fileset(dir: projectDir) {
+                include(name: "workbench/package.json")
+                include(name: "workbench/opensearch_dashboards.json")
+            }
+        }
+        ant.replaceregexp(file:'workbench/opensearch_dashboards.json', match:'"opensearchDashboardsVersion": "\\d+.\\d+.\\d+', replace:'"opensearchDashboardsVersion": ' + '"' + newVersion.tokenize('-')[0], flags:'g', byline:true)
+    }
+}

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -27,16 +27,54 @@ public static String unquote(String text, String mark) {
 
   /**
    * Unquote Identifier which has " or ' or ` as mark.
+   * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote
+   * acts as an escape
+   * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced
+   * with double.
    * @param text string
    * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been
    *     removed
    */
   public static String unquoteText(String text) {
-    if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) {
-      return text.substring(1, text.length() - 1);
+
+    if (text.length() < 2) {
+      return text;
+    }
+
+    char enclosingQuote;
+    char firstChar = text.charAt(0);
+    char lastChar = text.charAt(text.length() - 1);
+
+    if (firstChar == lastChar
+        && (firstChar == '\''
+        || firstChar == '"'
+        || firstChar == '`')) {
+      enclosingQuote = firstChar;
     } else {
       return text;
     }
+
+    if (enclosingQuote == '`') {
+      return text.substring(1, text.length() - 1);
+    }
+
+    char currentChar;
+    char nextChar;
+
+    StringBuilder textSB = new StringBuilder();
+
+    // Ignores first and last character as they are the quotes that should be removed
+    for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) {
+      currentChar = text.charAt(chIndex);
+      nextChar = text.charAt(chIndex + 1);
+      if (currentChar == enclosingQuote
+          && nextChar == currentChar) {
+        chIndex++;
+      }
+      textSB.append(currentChar);
+    }
+
+    return textSB.toString();
   }
 
   /**

core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java

@@ -0,0 +1,48 @@
+package org.opensearch.sql.common.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.opensearch.sql.common.utils.StringUtils.unquoteText;
+
+import org.junit.jupiter.api.Test;
+
+class StringUtilsTest {
+  @Test
+  void unquoteTest() {
+    assertEquals("test", unquoteText("test"));
+    assertEquals("test", unquoteText("'test'"));
+    assertEquals("test", unquoteText("`test`"));
+
+    assertEquals("test'", unquoteText("'test'''"));
+    assertEquals("test\"", unquoteText("\"test\"\"\""));
+
+    assertEquals("te``st", unquoteText("'te``st'"));
+    assertEquals("te``st", unquoteText("\"te``st\""));
+    assertEquals("te``st", unquoteText("`te``st`"));
+
+    assertEquals("te'st", unquoteText("'te''st'"));
+    assertEquals("te''st", unquoteText("\"te''st\""));
+    assertEquals("te''st", unquoteText("`te''st`"));
+
+    assertEquals("te\"\"st", unquoteText("'te\"\"st'"));
+    assertEquals("te\"st", unquoteText("\"te\"\"st\""));
+    assertEquals("te\"\"st", unquoteText("`te\"\"st`"));
+
+    assertEquals("''", unquoteText("''''''"));
+    assertEquals("\"\"", unquoteText("\"\"\"\"\"\""));
+    assertEquals("````", unquoteText("``````"));
+
+    assertEquals("test'", unquoteText("'test''"));
+
+    assertEquals("", unquoteText(""));
+    assertEquals("'", unquoteText("'"));
+    assertEquals("`", unquoteText("`"));
+    assertEquals("\"", unquoteText("\""));
+
+    assertEquals("hello'", unquoteText("'hello''"));
+    assertEquals("don't", unquoteText("'don't'"));
+    assertEquals("hello`", unquoteText("`hello``"));
+    assertEquals("don\"t", unquoteText("\"don\"t\""));
+
+  }
+
+}

integ-test/build.gradle

@@ -187,6 +187,10 @@ task compileJdbc(type: Exec) {
     }
 }
 
+/*
+BWC test suite was running on OpenDistro which was discontinued and no available anymore for testing.
+Test suite is not removed, because it could be reused later between different OpenSearch versions.
+*/
 String bwcVersion = "1.13.2.0";
 String baseName = "sqlBwcCluster"
 String bwcFilePath = "src/test/resources/bwc/"

integ-test/src/test/java/org/opensearch/sql/ppl/MatchIT.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+
+public class MatchIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.BANK);
+  }
+
+  @Test
+  public void test_match_function() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | where match(firstname, 'Hattie') | fields firstname",
+                TEST_INDEX_BANK));
+    verifyDataRows(result, rows("Hattie"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PHRASE;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class MatchPhraseIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.PHRASE);
+  }
+
+  @Test
+  public void test_match_phrase_function() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where match_phrase(phrase, 'quick fox') | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("quick fox"), rows("quick fox here"));
+  }
+
+  @Test
+  @Ignore("Not supported actually in PPL")
+  public void test_matchphrase_legacy_function() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where matchphrase(phrase, 'quick fox') | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("quick fox"), rows("quick fox here"));
+  }
+
+  @Test
+  public void test_match_phrase_with_slop() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where match_phrase(phrase, 'brown fox', slop = 2) | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("brown fox"), rows("fox brown"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhrasePrefixWhereCommandIT.java → integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhrasePrefixIT.java

@@ -13,7 +13,7 @@
 import org.json.JSONObject;
 import org.junit.Test;
 
-public class MatchPhrasePrefixWhereCommandIT extends PPLIntegTestCase {
+public class MatchPhrasePrefixIT extends PPLIntegTestCase {
 
   @Override
   public void init() throws IOException {
@@ -93,7 +93,7 @@ public void zero_term_query_all() throws IOException {
 
   @Test
   public void slop_is_2() throws IOException {
-    // When slop is 0, the terms are matched exactly in the order specified.
+    // When slop is 2, the terms are matched exactly in the order specified.
     // 'open' is used to match prefix of the next term.
     String query = "source = %s" +
         "| where match_phrase_prefix(Tags, 'gas ta', slop=2) " +
@@ -104,7 +104,7 @@ public void slop_is_2() throws IOException {
 
   @Test
   public void slop_is_3() throws IOException {
-    // When slop is 2, results will include phrases where the query terms are transposed.
+    // When slop is 3, results will include phrases where the query terms are transposed.
     String query = "source = %s" +
         "| where match_phrase_prefix(Tags, 'gas ta', slop=3)" +
         "| fields Tags";

integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER;
+
+import java.io.IOException;
+
+import org.json.JSONObject;
+import org.junit.Test;
+
+public class MultiMatchIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.BEER);
+  }
+
+  @Test
+  public void test_multi_match() throws IOException {
+    String query = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
+    var result = executeQuery(query);
+    assertEquals(16, result.getInt("total"));
+  }
+
+  @Test
+  public void test_multi_match_all_params() throws IOException {
+    String query = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['Body', Tags], 'taste beer', operator='and', analyzer=english,"
+        + "auto_generate_synonyms_phrase_query=true, boost = 0.77, cutoff_frequency=0.33,"
+        + "fuzziness = 'AUTO:1,5', fuzzy_transpositions = false, lenient = true, max_expansions = 25,"
+        + "minimum_should_match = '2<-25% 9<-3', prefix_length = 7, tie_breaker = 0.3,"
+        + "type = most_fields, slop = 2, zero_terms_query = 'ALL') | fields Id";
+    var result = executeQuery(query);
+    assertEquals(10, result.getInt("total"));
+  }
+
+  @Test
+  public void test_wildcard_multi_match() throws IOException {
+    String query1 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['Tags'], 'taste') | fields Id";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['T*'], 'taste') | fields Id";
+    var result2 = executeQuery(query2);
+    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
+
+    String query3 = "source=" + TEST_INDEX_BEER
+        + " | where simple_query_string(['*Date'], '2014-01-22')";
+    JSONObject result3 = executeQuery(query3);
+    assertEquals(10, result3.getInt("total"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java

@@ -22,14 +22,14 @@ public void init() throws IOException {
   public void all_fields_test() throws IOException {
     String query = "source=" + TEST_INDEX_BEER + " | where query_string([`*`], 'taste')";
     JSONObject result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+    assertEquals(16, result.getInt("total"));
   }
 
   @Test
   public void mandatory_params_test() throws IOException {
     String query = "source=" + TEST_INDEX_BEER + " | where query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
     JSONObject result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+    assertEquals(16, result.getInt("total"));
   }
 
   @Test
@@ -44,7 +44,7 @@ public void all_params_test() throws IOException {
         + "fuzzy_transpositions = false, lenient = true, fuzzy_max_expansions = 25,"
         + "minimum_should_match = '2<-25% 9<-3', fuzzy_prefix_length = 7)";
     JSONObject result = executeQuery(query);
-    assertEquals(1990, result.getInt("total"));
+    assertEquals(49, result.getInt("total"));
   }
 
   @Test
@@ -57,8 +57,8 @@ public void wildcard_test() throws IOException {
     JSONObject result2 = executeQuery(query2);
     assertNotEquals(result1.getInt("total"), result2.getInt("total"));
 
-    String query3 = "source=" + TEST_INDEX_BEER + " | where query_string(['*Date'], '2015-01-29')";
+    String query3 = "source=" + TEST_INDEX_BEER + " | where query_string(['*Date'], '2014-01-22')";
     JSONObject result3 = executeQuery(query3);
-    assertEquals(5, result3.getInt("total"));
+    assertEquals(10, result3.getInt("total"));
   }
 }