Sync our main with upstream/2.x

.github/workflows/bi-connectors.yml

@@ -2,7 +2,6 @@ name: Build connectors for BI tools
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     paths:
       - 'bi-connectors/PowerBIConnector/**'

.github/workflows/codeql-analysis.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - 'main'
       - '[1-9]+.[0-9x]+'
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'

.github/workflows/draft-release-notes-workflow.yml

@@ -2,7 +2,6 @@ name: Release Drafter
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches:
       - 'main'

.github/workflows/link-checker.yml

@@ -2,7 +2,6 @@ name: Link Checker
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches:
       - 'main'

.github/workflows/sql-cli-test-and-build-workflow.yml

@@ -2,7 +2,6 @@ name: SQL CLI Test and Build
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'

.github/workflows/sql-jdbc-test-and-build-workflow.yml

@@ -2,7 +2,6 @@ name: SQL JDBC Java CI
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'

.github/workflows/sql-odbc-main.yml

@@ -2,7 +2,6 @@ name: OpenSearch ODBC Driver
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'

.github/workflows/sql-test-and-build-workflow.yml

@@ -2,7 +2,6 @@ name: SQL Java CI
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'
@@ -21,25 +20,32 @@ on:
 jobs:
   build:
     strategy:
+      # Run all jobs
+      fail-fast: false
       matrix:
-        java:
-          - 11
-          - 17
-    runs-on: ubuntu-latest
+        entry:
+          - { os: ubuntu-latest, java: 11 }
+          - { os: windows-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc}
+          - { os: macos-latest, java: 11, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
+          - { os: ubuntu-latest, java: 17 }
+          - { os: windows-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
+          - { os: macos-latest, java: 17, os_build_args: -x doctest -x integTest -x jacocoTestReport -x compileJdbc }
+    runs-on: ${{ matrix.entry.os }}
 
     steps:
     - uses: actions/checkout@v3
-    
+
     - name: Set up JDK ${{ matrix.java }}
       uses: actions/setup-java@v3
       with:
         distribution: 'temurin'
-        java-version: ${{ matrix.java }}
-    
+        java-version: ${{ matrix.entry.java }}
+
     - name: Build with Gradle
-      run: ./gradlew --continue build assemble
+      run: ./gradlew --continue build ${{ matrix.entry.os_build_args }}
 
     - name: Run backward compatibility tests
+      if: ${{ matrix.entry.os == 'ubuntu-latest' }}
       run: ./scripts/bwctest.sh
 
     - name: Create Artifact Path
@@ -49,7 +55,7 @@ jobs:
 
     # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
     - name: Upload SQL Coverage Report
-      if: always()
+      if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
       uses: codecov/codecov-action@v3
       with:
         flags: sql-engine
@@ -58,11 +64,11 @@ jobs:
     - name: Upload Artifacts
       uses: actions/upload-artifact@v2
       with:
-        name: opensearch-sql
+        name: opensearch-sql-${{ matrix.entry.os }}
         path: opensearch-sql-builds
 
     - name: Upload test reports
-      if: always()
+      if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
       uses: actions/upload-artifact@v2
       with:
         name: test-reports

.github/workflows/sql-test-workflow.yml

@@ -2,6 +2,13 @@ name: SQL Plugin Tests
 
 on:
   workflow_dispatch:
+    inputs:
+      name:
+        required: false
+        type: string
+
+run-name:
+  ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }}
 
 jobs:
   build:
@@ -64,10 +71,10 @@ jobs:
 
     - name: Verify test results
       run: |
-        if [[ -e failures.log ]]
+        if [[ -e report.log ]]
         then
           echo "## FAILED TESTS :facepalm::warning::bangbang:" >> $GITHUB_STEP_SUMMARY
-          cat failures.log >> $GITHUB_STEP_SUMMARY
+          cat report.log >> $GITHUB_STEP_SUMMARY
           exit 1
         fi
 

.github/workflows/sql-workbench-test-and-build-workflow.yml

@@ -2,7 +2,6 @@ name: SQL Workbench Test and Build
 
 on:
   pull_request:
-    types: [opened, reopened]
   push:
     branches-ignore:
       - 'dependabot/**'
@@ -17,8 +16,15 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
     steps:
+      - name: Enable longer filenames
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: git config --system core.longpaths true
+
       - name: Checkout Plugin
         uses: actions/checkout@v3
 
@@ -52,7 +58,7 @@ jobs:
           yarn test:jest --coverage
 
       - name: Upload coverage
-        if: always()
+        if: ${{ always() && matrix.os == 'ubuntu-latest' }}
         uses: codecov/codecov-action@v3
         with:
           flags: query-workbench
@@ -69,5 +75,6 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v1 # can't update to v3 because upload fails
         with:
-          name: workbench
+          name: workbench-${{ matrix.os }}
           path: ../OpenSearch-Dashboards/plugins/workbench/build
+

.gitignore

@@ -42,3 +42,5 @@ gen
 .DS_Store
 
 /artifacts/
+/.pid.lock
+/.prom.pid.lock

NOTICE

@@ -11,6 +11,9 @@ Foundation (http://www.apache.org/).
 This product includes software developed by
 Joda.org (http://www.joda.org/).
 
+This product includes software developed by
+Kraken (https://github.com/thekrakken/java-grok).
+
 This project is based on the Apache 2.0-licensed elasticsearch-sql project (https://github.com/NLPchina/elasticsearch-sql):
 
 Copyright 2014 omershelef

common/build.gradle

@@ -35,6 +35,10 @@ dependencies {
     api "org.antlr:antlr4-runtime:4.7.1"
     api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1'
+    api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
+    testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'
+    testImplementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
+    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
 }

common/src/main/java/org/opensearch/sql/common/grok/Converter.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.common.grok;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Convert String argument to the right type.
+ */
+public class Converter {
+
+  public enum Type {
+    BYTE(Byte::valueOf),
+    BOOLEAN(Boolean::valueOf),
+    SHORT(Short::valueOf),
+    INT(Integer::valueOf, "integer"),
+    LONG(Long::valueOf),
+    FLOAT(Float::valueOf),
+    DOUBLE(Double::valueOf),
+    DATETIME(new DateConverter(), "date"),
+    STRING(v -> v, "text");
+
+    public final IConverter<? extends Object> converter;
+    public final List<String> aliases;
+
+    Type(IConverter<? extends Object> converter, String... aliases) {
+      this.converter = converter;
+      this.aliases = Arrays.asList(aliases);
+    }
+  }
+
+  private static final Pattern SPLITTER = Pattern.compile("[:;]");
+
+  private static final Map<String, Type> TYPES =
+      Arrays.stream(Type.values())
+          .collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t));
+
+  private static final Map<String, Type> TYPE_ALIASES =
+      Arrays.stream(Type.values())
+          .flatMap(type -> type.aliases.stream()
+              .map(alias -> new AbstractMap.SimpleEntry<>(alias, type)))
+          .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+  private static Type getType(String key) {
+    key = key.toLowerCase();
+    Type type = TYPES.getOrDefault(key, TYPE_ALIASES.get(key));
+    if (type == null) {
+      throw new IllegalArgumentException("Invalid data type :" + key);
+    }
+    return type;
+  }
+
+  /**
+   * getConverters.
+   */
+  public static Map<String, IConverter<? extends Object>>
+      getConverters(Collection<String> groupNames, Object... params) {
+    return groupNames.stream()
+        .filter(Converter::containsDelimiter)
+        .collect(Collectors.toMap(Function.identity(), key -> {
+          String[] list = splitGrokPattern(key);
+          IConverter<? extends Object> converter = getType(list[1]).converter;
+          if (list.length == 3) {
+            converter = converter.newConverter(list[2], params);
+          }
+          return converter;
+        }));
+  }
+
+  /**
+   * getGroupTypes.
+   */
+  public static Map<String, Type> getGroupTypes(Collection<String> groupNames) {
+    return groupNames.stream()
+        .filter(Converter::containsDelimiter)
+        .map(Converter::splitGrokPattern)
+        .collect(Collectors.toMap(
+            l -> l[0],
+            l -> getType(l[1])
+        ));
+  }
+
+  public static String extractKey(String key) {
+    return splitGrokPattern(key)[0];
+  }
+
+  private static boolean containsDelimiter(String string) {
+    return string.indexOf(':') >= 0 || string.indexOf(';') >= 0;
+  }
+
+  private static String[] splitGrokPattern(String string) {
+    return SPLITTER.split(string, 3);
+  }
+
+  interface IConverter<T> {
+
+    T convert(String value);
+
+    default IConverter<T> newConverter(String param, Object... params) {
+      return this;
+    }
+  }
+
+
+  static class DateConverter implements IConverter<Instant> {
+
+    private final DateTimeFormatter formatter;
+    private final ZoneId timeZone;
+
+    public DateConverter() {
+      this.formatter = DateTimeFormatter.ISO_DATE_TIME;
+      this.timeZone = ZoneOffset.UTC;
+    }
+
+    private DateConverter(DateTimeFormatter formatter, ZoneId timeZone) {
+      this.formatter = formatter;
+      this.timeZone = timeZone;
+    }
+
+    @Override
+    public Instant convert(String value) {
+      TemporalAccessor dt = formatter
+          .parseBest(value.trim(), ZonedDateTime::from, LocalDateTime::from, OffsetDateTime::from,
+              Instant::from,
+              LocalDate::from);
+      if (dt instanceof ZonedDateTime) {
+        return ((ZonedDateTime) dt).toInstant();
+      } else if (dt instanceof LocalDateTime) {
+        return ((LocalDateTime) dt).atZone(timeZone).toInstant();
+      } else if (dt instanceof OffsetDateTime) {
+        return ((OffsetDateTime) dt).atZoneSameInstant(timeZone).toInstant();
+      } else if (dt instanceof Instant) {
+        return ((Instant) dt);
+      } else if (dt instanceof LocalDate) {
+        return ((LocalDate) dt).atStartOfDay(timeZone).toInstant();
+      } else {
+        return null;
+      }
+    }
+
+    @Override
+    public DateConverter newConverter(String param, Object... params) {
+      if (!(params.length == 1 && params[0] instanceof ZoneId)) {
+        throw new IllegalArgumentException("Invalid parameters");
+      }
+      return new DateConverter(DateTimeFormatter.ofPattern(param), (ZoneId) params[0]);
+    }
+  }
+}