Sql add support for geo shape type (#4204)

Adds initial very minimal support for geo_shapes to SQL. For now, geoshapes are converted into String instead of Geometry objects on the JDBC side and no effort to parse the result is performed. Relates #4080
elastic · May 9, 2018 · 1157c11 · 1157c11
1 parent 54122d8
commit 1157c11
Show file tree

Hide file tree

Showing 24 changed files with 1,286 additions and 30 deletions.
diff --git a/...ck/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/JdbcResultSet.java b/...ck/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/JdbcResultSet.java
@@ -367,9 +367,9 @@ private <T> T convert(int columnIndex, Class<T> type) throws SQLException {
             }
         }
 
-        JDBCType columnType = cursor.columns().get(columnIndex - 1).type;
-        
-        return TypeConverter.convert(val, columnType, type);
+        ColumnInfo columnInfo = cursor.columns().get(columnIndex - 1);
+
+        return TypeConverter.convert(val, columnInfo.type, columnInfo.esType, type);
     }
 
     @Override

diff --git a/...ck/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverter.java b/...ck/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverter.java
@@ -99,12 +99,12 @@ private static <T> T dateTimeConvert(Long millis, Calendar c, Function<Calendar,
      * Converts object val from columnType to type
      */
     @SuppressWarnings("unchecked")
-    static <T> T convert(Object val, JDBCType columnType, Class<T> type) throws SQLException {
+    static <T> T convert(Object val, JDBCType columnType, String esType, Class<T> type) throws SQLException {
         if (type == null) {
-            return (T) convert(val, columnType);
+            return (T) convert(val, columnType, esType);
         }
         if (type == String.class) {
-            return (T) asString(convert(val, columnType));
+            return (T) asString(convert(val, columnType, esType));
         }
         if (type == Boolean.class) {
             return (T) asBoolean(val, columnType);
@@ -185,7 +185,7 @@ public static String classNameOf(JDBCType jdbcType) throws JdbcSQLException {
      * <p>
      * The returned types needs to correspond to ES-portion of classes returned by {@link TypeConverter#classNameOf}
      */
-    static Object convert(Object v, JDBCType columnType) throws SQLException {
+    static Object convert(Object v, JDBCType columnType, String esType) throws SQLException {
         switch (columnType) {
             case NULL:
                 return null;
@@ -207,9 +207,20 @@ static Object convert(Object v, JDBCType columnType) throws SQLException {
                 return floatValue(v);  // Float might be represented as string for infinity and NaN values
             case TIMESTAMP:
                 return ((Number) v).longValue();
+            case OTHER:
+                return convertOtherType(esType, v);
             default:
                 throw new SQLException("Unexpected column type [" + columnType.getName() + "]");
+        }
+    }
 
+    static Object convertOtherType(String esType, Object v) throws SQLException  {
+        switch (esType) {
+            case "geo_shape":
+                // TODO: convert this into a geo object
+                return v;
+            default:
+                throw new SQLException("Unsupported es type [" + esType + "]");
         }
     }
 

diff --git a/...in/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/client/JdbcHttpClient.java b/...in/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/client/JdbcHttpClient.java
@@ -86,7 +86,8 @@ private InfoResponse fetchServerInfo() throws SQLException {
      */
     private List<ColumnInfo> toJdbcColumnInfo(List<org.elasticsearch.xpack.sql.plugin.ColumnInfo> columns) {
         return columns.stream().map(columnInfo ->
-                new ColumnInfo(columnInfo.name(), columnInfo.jdbcType(), EMPTY, EMPTY, EMPTY, EMPTY, columnInfo.displaySize())
+                new ColumnInfo(columnInfo.name(), columnInfo.jdbcType(), EMPTY, EMPTY, EMPTY, EMPTY, columnInfo.displaySize(),
+                        columnInfo.esType())
         ).collect(Collectors.toList());
     }
 }
diff --git a/...ugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/protocol/ColumnInfo.java b/...ugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/net/protocol/ColumnInfo.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.sql.jdbc.net.protocol;
 
+import org.elasticsearch.xpack.sql.type.DataType;
+
 import java.sql.JDBCType;
 import java.util.Objects;
 
@@ -16,8 +18,14 @@ public class ColumnInfo {
     public final String name;
     public final int displaySize;
     public final JDBCType type;
+    public final String esType;
 
     public ColumnInfo(String name, JDBCType type, String table, String catalog, String schema, String label, int displaySize) {
+        this(name, type, table, catalog, schema, label, displaySize, DataType.fromJdbcType(type).esType);
+    }
+
+    public ColumnInfo(String name, JDBCType type, String table, String catalog, String schema, String label, int displaySize,
+                      String esType) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null");
         }
@@ -36,13 +44,17 @@ public ColumnInfo(String name, JDBCType type, String table, String catalog, Stri
         if (label == null) {
             throw new IllegalArgumentException("[label] must not be null");
         }
+        if (esType == null) {
+            throw new IllegalArgumentException("[esType] must not be null");
+        }
         this.name = name;
         this.type = type;
         this.table = table;
         this.catalog = catalog;
         this.schema = schema;
         this.label = label;
         this.displaySize = displaySize;
+        this.esType = esType;
     }
 
     public int displaySize() {
@@ -81,11 +93,12 @@ public boolean equals(Object obj) {
                 && catalog.equals(other.catalog)
                 && schema.equals(other.schema)
                 && label.equals(other.label)
-                && displaySize == other.displaySize;
+                && displaySize == other.displaySize
+                && esType.equals(other.esType);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, type, table, catalog, schema, label, displaySize);
+        return Objects.hash(name, type, table, catalog, schema, label, displaySize, esType);
     }
 }
diff --git a/...ugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverterTests.java b/...ugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/jdbc/TypeConverterTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.sql.plugin.AbstractSqlRequest;
 import org.elasticsearch.xpack.sql.plugin.SqlQueryResponse;
+import org.elasticsearch.xpack.sql.type.DataType;
 import org.joda.time.DateTime;
 
 import java.sql.JDBCType;
@@ -55,7 +56,7 @@ private Object convertAsNative(Object value, JDBCType type) throws Exception {
         builder.endObject();
         builder.close();
         Object copy = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value");
-        return TypeConverter.convert(copy, type);
+        return TypeConverter.convert(copy, type, DataType.fromJdbcType(type).esType);
     }
 
 }
diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java
@@ -37,14 +37,16 @@ public enum DataType {
     OBJECT(      JDBCType.STRUCT,    null,            -1,                0,                 0),
     NESTED(      JDBCType.STRUCT,    null,            -1,                0,                 0),
     BINARY(      JDBCType.VARBINARY, byte[].class,    -1,                Integer.MAX_VALUE, 0),
-    DATE(        JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES,        19,                20);
+    DATE(        JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES,        19,                20),
+    // TODO: This should map to some Geography class instead of String
+    GEO_SHAPE(   JDBCType.OTHER,     String.class,    Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false);
     // @formatter:on
 
     private static final Map<JDBCType, DataType> jdbcToEs;
 
     static {
         jdbcToEs = Arrays.stream(DataType.values())
-                .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT) // Remove duplicates
+                .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE)
                 .collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
     }
 

diff --git a/...ql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java b/...ql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java
@@ -58,11 +58,11 @@ public void testSysTypes() throws Exception {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "SHORT", "INTEGER", "LONG", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", "KEYWORD", "TEXT",
-                "DATE", "BINARY", "NULL", "UNSUPPORTED", "OBJECT", "NESTED", "BOOLEAN");
+                "DATE", "BINARY", "NULL", "UNSUPPORTED",  "GEO_SHAPE", "OBJECT", "NESTED", "BOOLEAN");
 
         cmd.execute(null, ActionListener.wrap(r -> {
             assertEquals(19, r.columnCount());
-            assertEquals(17, r.size());
+            assertEquals(18, r.size());
             assertFalse(r.schema().types().contains(DataType.NULL));
             // test numeric as signed
             assertFalse(r.column(9, Boolean.class));

diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java
@@ -15,6 +15,7 @@
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.xpack.sql.type.DataType.DATE;
+import static org.elasticsearch.xpack.sql.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.sql.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD;
 import static org.elasticsearch.xpack.sql.type.DataType.NESTED;
@@ -38,12 +39,13 @@ public void testEmptyMap() {
 
     public void testBasicMapping() {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
-        assertThat(mapping.size(), is(6));
+        assertThat(mapping.size(), is(7));
         assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
         assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
         assertThat(mapping.get("last_name").getDataType(), is(TEXT));
         assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
         assertThat(mapping.get("salary").getDataType(), is(INTEGER));
+        assertThat(mapping.get("site").getDataType(), is(GEO_SHAPE));
     }
 
     public void testDefaultStringMapping() {

diff --git a/x-pack/plugin/sql/src/test/resources/mapping-basic.json b/x-pack/plugin/sql/src/test/resources/mapping-basic.json
@@ -17,6 +17,9 @@
         },
         "salary" : {
             "type" : "integer"
+        },
+        "site": {
+            "type": "geo_shape"
         }
     }
 }
diff --git a/x-pack/qa/sql/build.gradle b/x-pack/qa/sql/build.gradle
@@ -15,6 +15,7 @@ dependencies {
   // CLI testing dependencies
   compile project(path: xpackModule('sql:sql-cli'), configuration: 'nodeps')
   compile "org.jline:jline:3.6.0"
+  compile "org.orbisgis:h2gis-ext:1.3.2"
 }
 
 /* disable unit tests because these are all integration tests used
@@ -69,6 +70,9 @@ thirdPartyAudit.excludes = [
     'org.fusesource.jansi.internal.Kernel32$SMALL_RECT',
     'org.fusesource.jansi.internal.Kernel32',
     'org.fusesource.jansi.internal.WindowsSupport',
+    // TODO: Temporary solution until core removes JTS as a dependency
+    'org.h2gis.functions.factory.H2GISFunctions',
+    'org.h2gis.network.functions.NetworkFunctions',
     'org.mozilla.universalchardet.UniversalDetector',
 ]
 
@@ -85,9 +89,16 @@ subprojects {
     testCompile "org.elasticsearch.test:framework:${version}"
 
     // JDBC testing dependencies
-    testRuntime "net.sourceforge.csvjdbc:csvjdbc:1.0.34"
-    testRuntime "com.h2database:h2:1.4.197"
     testRuntime xpackProject('plugin:sql:jdbc')
+    testRuntime("net.sourceforge.csvjdbc:csvjdbc:1.0.34") {
+      transitive = false
+    }
+
+    testRuntime("com.h2database:h2:1.4.196") {
+      transitive = false
+    }
+
+    testRuntime "org.orbisgis:h2gis-ext:1.3.2"
 
     // TODO check if needed
     testRuntime("org.antlr:antlr4-runtime:4.5.3") {
@@ -99,6 +110,16 @@ subprojects {
     testRuntime "org.jline:jline:3.6.0"
   }
 
+  // TODO: Temporary solution until core removes JTS as a dependency
+  configurations {
+    testRuntime {
+      exclude group: 'org.locationtech.jts', module: 'jts-core'
+    }
+    testCompile {
+      exclude group: 'org.locationtech.jts', module: 'jts-core'
+    }
+  }
+
   if (project.name != 'security') {
     // The security project just configures its subprojects
     apply plugin: 'elasticsearch.rest-test'

diff --git a/...no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcCsvSpecIT.java b/...no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcCsvSpecIT.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.qa.sql.nosecurity;
+
+import org.elasticsearch.xpack.qa.sql.geo.GeoCsvSpecTestCase;
+import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+
+public class GeoJdbcCsvSpecIT extends GeoCsvSpecTestCase {
+    public GeoJdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+        super(fileName, groupName, testName, lineNumber, testCase);
+    }
+}
diff --git a/...no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcSqlSpecIT.java b/...no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/GeoJdbcSqlSpecIT.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.qa.sql.nosecurity;
+
+import org.elasticsearch.xpack.qa.sql.geo.GeoSqlSpecTestCase;
+
+public class GeoJdbcSqlSpecIT extends GeoSqlSpecTestCase {
+    public GeoJdbcSqlSpecIT(String fileName, String groupName, String testName, Integer lineNumber, String query) {
+        super(fileName, groupName, testName, lineNumber, query);
+    }
+}
diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.qa.sql.geo;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
+import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
+import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
+import org.junit.Before;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.csvConnection;
+import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.executeCsvQuery;
+import static org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.specParser;
+
+/**
+ * Tests comparing sql queries executed against our jdbc client
+ * with hard coded result sets.
+ */
+public abstract class GeoCsvSpecTestCase extends SpecBaseIntegrationTestCase {
+    private final CsvTestCase testCase;
+
+    @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readScriptSpec() throws Exception {
+        Parser parser = specParser();
+        List<Object[]> tests = new ArrayList<>();
+        tests.addAll(readScriptSpec("/ogc/ogc.csv-spec", parser));
+        return tests;
+    }
+
+    public GeoCsvSpecTestCase(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+        super(fileName, groupName, testName, lineNumber);
+        this.testCase = testCase;
+    }
+
+
+    @Before
+    public void setupTestGeoDataIfNeeded() throws Exception {
+        if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
+            GeoDataLoader.loadDatasetIntoEs(client());
+        }
+    }
+
+    @Override
+    protected final void doTest() throws Throwable {
+        try (Connection csv = csvConnection(testCase.expectedResults);
+             Connection es = esJdbc()) {
+
+            // pass the testName as table for debugging purposes (in case the underlying reader is missing)
+            ResultSet expected = executeCsvQuery(csv, testName);
+            ResultSet elasticResults = executeJdbcQuery(es, testCase.query);
+            assertResults(expected, elasticResults);
+        }
+    }
+
+    // make sure ES uses UTC (otherwise JDBC driver picks up the JVM timezone per spec/convention)
+    @Override
+    protected Properties connectionProperties() {
+        Properties connectionProperties = new Properties();
+        connectionProperties.setProperty(JdbcConfiguration.TIME_ZONE, "UTC");
+        return connectionProperties;
+    }
+
+}