SQL: Add basic support for geo point (#31257)

Adds basic support for geo point type. For now, the geopoint is
represented as a string and returned in the same format it was stored
in the source.

Relates to #29872

x-pack/plugin/sql/sql-shared-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

@@ -42,14 +42,16 @@ public enum DataType {
     // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288
     DATE(        JDBCType.TIMESTAMP, Timestamp.class, Long.BYTES,        24,                24),
     // TODO: This should map to some Geography class instead of String
-    GEO_SHAPE(   JDBCType.OTHER,     String.class,    Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false);
+    GEO_SHAPE(   JDBCType.OTHER,     String.class,    Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false),
+    GEO_POINT(   JDBCType.OTHER,     String.class,    Double.BYTES*2,    Integer.MAX_VALUE, 0, false, false, false);
     // @formatter:on
 
     private static final Map<JDBCType, DataType> jdbcToEs;
 
     static {
         jdbcToEs = Arrays.stream(DataType.values())
-                .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE)
+                .filter(dataType -> dataType != TEXT && dataType != NESTED && dataType != SCALED_FLOAT && dataType != GEO_SHAPE &&
+                    dataType != GEO_POINT)
                 .collect(Collectors.toMap(dataType -> dataType.jdbcType, dataType -> dataType));
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java

@@ -58,11 +58,11 @@ public void testSysTypes() throws Exception {
         Command cmd = sql("SYS TYPES").v1();
 
         List<String> names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE",
-                "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "OBJECT", "NESTED");
+                "KEYWORD", "TEXT", "BOOLEAN", "DATE", "UNSUPPORTED", "GEO_SHAPE", "GEO_POINT", "OBJECT", "NESTED");
 
         cmd.execute(null, ActionListener.wrap(r -> {
             assertEquals(19, r.columnCount());
-            assertEquals(18, r.size());
+            assertEquals(names.size(), r.size());
             assertFalse(r.schema().types().contains(DataType.NULL));
             // test numeric as signed
             assertFalse(r.column(9, Boolean.class));

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java

@@ -15,6 +15,7 @@
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.xpack.sql.type.DataType.DATE;
+import static org.elasticsearch.xpack.sql.type.DataType.GEO_POINT;
 import static org.elasticsearch.xpack.sql.type.DataType.GEO_SHAPE;
 import static org.elasticsearch.xpack.sql.type.DataType.INTEGER;
 import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD;
@@ -39,13 +40,12 @@ public void testEmptyMap() {
 
     public void testBasicMapping() {
         Map<String, EsField> mapping = loadMapping("mapping-basic.json");
-        assertThat(mapping.size(), is(7));
+        assertThat(mapping.size(), is(6));
         assertThat(mapping.get("emp_no").getDataType(), is(INTEGER));
         assertThat(mapping.get("first_name"), instanceOf(TextEsField.class));
         assertThat(mapping.get("last_name").getDataType(), is(TEXT));
         assertThat(mapping.get("gender").getDataType(), is(KEYWORD));
         assertThat(mapping.get("salary").getDataType(), is(INTEGER));
-        assertThat(mapping.get("site").getDataType(), is(GEO_SHAPE));
     }
 
     public void testDefaultStringMapping() {
@@ -181,8 +181,11 @@ public void testNestedDoc() {
 
     public void testGeoField() {
         Map<String, EsField> mapping = loadMapping("mapping-geo.json");
-        EsField dt = mapping.get("location");
-        assertThat(dt.getDataType().esType, is("unsupported"));
+        assertThat(mapping.size(), is(2));
+        EsField gp = mapping.get("location");
+        assertThat(gp.getDataType().esType, is("geo_point"));
+        EsField gs = mapping.get("site");
+        assertThat(gs.getDataType().esType, is("geo_shape"));
     }
 
     public void testUnsupportedTypes() {
@@ -202,4 +205,4 @@ public static Map<String, EsField> loadMapping(String name, boolean ordered) {
         assertNotNull("Could not find mapping resource:" + name, stream);
         return Types.fromEs(XContentHelper.convertToMap(JsonXContent.jsonXContent, stream, ordered));
     }
-}
+}

x-pack/plugin/sql/src/test/resources/mapping-basic.json

@@ -17,9 +17,6 @@
         },
         "salary" : {
             "type" : "integer"
-        },
-        "site": {
-            "type": "geo_shape"
         }
     }
 }

x-pack/plugin/sql/src/test/resources/mapping-geo.json

@@ -2,6 +2,9 @@
     "properties" : {
         "location" : {
             "type" : "geo_point"
+        },
+        "site": {
+            "type" : "geo_shape"
         }
     }
 }

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoCsvSpecTestCase.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.qa.sql.geo;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.xpack.qa.sql.jdbc.CsvTestUtils.CsvTestCase;
 import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
 import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
@@ -34,6 +35,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
         Parser parser = specParser();
         List<Object[]> tests = new ArrayList<>();
         tests.addAll(readScriptSpec("/ogc/ogc.csv-spec", parser));
+        tests.addAll(readScriptSpec("/geo/geosql.csv-spec", parser));
         return tests;
     }
 
@@ -45,8 +47,11 @@ public GeoCsvSpecTestCase(String fileName, String groupName, String testName, In
 
     @Before
     public void setupTestGeoDataIfNeeded() throws Exception {
-        if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
-            GeoDataLoader.loadDatasetIntoEs(client());
+        if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) {
+            GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc");
+        }
+        if (client().performRequest(new Request("HEAD", "/geo")).getStatusLine().getStatusCode() == 404) {
+            GeoDataLoader.loadGeoDatasetIntoEs(client(), "geo");
         }
     }
 

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoDataLoader.java

@@ -11,6 +11,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
@@ -37,26 +38,28 @@ public class GeoDataLoader {
 
     public static void main(String[] args) throws Exception {
         try (RestClient client = RestClient.builder(new HttpHost("localhost", 9200)).build()) {
-            loadDatasetIntoEs(client);
+            loadOGCDatasetIntoEs(client, "ogc");
+            loadGeoDatasetIntoEs(client, "geo");
             Loggers.getLogger(GeoDataLoader.class).info("Geo data loaded");
         }
     }
 
-    protected static void loadDatasetIntoEs(RestClient client) throws Exception {
-        loadDatasetIntoEs(client, "ogc");
-        makeFilteredAlias(client, "lakes", "ogc", "\"term\" : { \"ogc_type\" : \"lakes\" }");
-        makeFilteredAlias(client, "road_segments", "ogc", "\"term\" : { \"ogc_type\" : \"road_segments\" }");
-        makeFilteredAlias(client, "divided_routes", "ogc", "\"term\" : { \"ogc_type\" : \"divided_routes\" }");
-        makeFilteredAlias(client, "forests", "ogc", "\"term\" : { \"ogc_type\" : \"forests\" }");
-        makeFilteredAlias(client, "bridges", "ogc", "\"term\" : { \"ogc_type\" : \"bridges\" }");
-        makeFilteredAlias(client, "streams", "ogc", "\"term\" : { \"ogc_type\" : \"streams\" }");
-        makeFilteredAlias(client, "buildings", "ogc", "\"term\" : { \"ogc_type\" : \"buildings\" }");
-        makeFilteredAlias(client, "ponds", "ogc", "\"term\" : { \"ogc_type\" : \"ponds\" }");
-        makeFilteredAlias(client, "named_places", "ogc", "\"term\" : { \"ogc_type\" : \"named_places\" }");
-        makeFilteredAlias(client, "map_neatlines", "ogc", "\"term\" : { \"ogc_type\" : \"map_neatlines\" }");
+    protected static void loadOGCDatasetIntoEs(RestClient client, String index) throws Exception {
+        createIndex(client, index, createOGCIndexRequest());
+        loadData(client, index, readResource("/ogc/ogc.json"));
+        makeFilteredAlias(client, "lakes", index, "\"term\" : { \"ogc_type\" : \"lakes\" }");
+        makeFilteredAlias(client, "road_segments", index, "\"term\" : { \"ogc_type\" : \"road_segments\" }");
+        makeFilteredAlias(client, "divided_routes", index, "\"term\" : { \"ogc_type\" : \"divided_routes\" }");
+        makeFilteredAlias(client, "forests", index, "\"term\" : { \"ogc_type\" : \"forests\" }");
+        makeFilteredAlias(client, "bridges", index, "\"term\" : { \"ogc_type\" : \"bridges\" }");
+        makeFilteredAlias(client, "streams", index, "\"term\" : { \"ogc_type\" : \"streams\" }");
+        makeFilteredAlias(client, "buildings", index, "\"term\" : { \"ogc_type\" : \"buildings\" }");
+        makeFilteredAlias(client, "ponds", index, "\"term\" : { \"ogc_type\" : \"ponds\" }");
+        makeFilteredAlias(client, "named_places", index, "\"term\" : { \"ogc_type\" : \"named_places\" }");
+        makeFilteredAlias(client, "map_neatlines", index, "\"term\" : { \"ogc_type\" : \"map_neatlines\" }");
     }
 
-    protected static void loadDatasetIntoEs(RestClient client, String index) throws Exception {
+    private static String createOGCIndexRequest() throws Exception {
         XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
         createIndex.startObject("settings");
         {
@@ -101,12 +104,21 @@ protected static void loadDatasetIntoEs(RestClient client, String index) throws
             createIndex.endObject();
         }
         createIndex.endObject().endObject();
+        return Strings.toString(createIndex);
+    }
 
-        client.performRequest("PUT", "/" + index, emptyMap(), new StringEntity(Strings.toString(createIndex),
-                        ContentType.APPLICATION_JSON));
+    private static void createIndex(RestClient client, String index, String settingsMappings) throws IOException {
+        Request createIndexRequest = new Request("PUT", "/" + index);
+        createIndexRequest.setEntity(new StringEntity(settingsMappings, ContentType.APPLICATION_JSON));
+        client.performRequest(createIndexRequest);
+    }
 
-        String bulk = readResource("/ogc/ogc.json");
+    static void loadGeoDatasetIntoEs(RestClient client, String index) throws Exception {
+        createIndex(client, index, readResource("/geo/geosql.json"));
+        loadData(client, index, readResource("/geo/geosql-bulk.json"));
+    }
 
+    private static void loadData(RestClient client, String index, String bulk) throws IOException {
         Response response = client.performRequest("POST", "/" + index + "/doc/_bulk", singletonMap("refresh", "true"),
                 new StringEntity(bulk, ContentType.APPLICATION_JSON));
 

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/geo/GeoSqlSpecTestCase.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.qa.sql.geo;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.xpack.qa.sql.jdbc.LocalH2;
 import org.elasticsearch.xpack.qa.sql.jdbc.SpecBaseIntegrationTestCase;
 import org.elasticsearch.xpack.sql.jdbc.jdbc.JdbcConfiguration;
@@ -47,8 +48,8 @@ public static List<Object[]> readScriptSpec() throws Exception {
     public void setupTestGeoDataIfNeeded() throws Exception {
         assumeTrue("Cannot support locales that don't use Hindu-Arabic numerals due to H2",
                 "42".equals(NumberFormat.getInstance(Locale.getDefault()).format(42)));
-        if (client().performRequest("HEAD", "/ogc").getStatusLine().getStatusCode() == 404) {
-            GeoDataLoader.loadDatasetIntoEs(client());
+        if (client().performRequest(new Request("HEAD", "/ogc")).getStatusLine().getStatusCode() == 404) {
+            GeoDataLoader.loadOGCDatasetIntoEs(client(), "ogc");
         }
     }
 

x-pack/qa/sql/src/main/resources/geo/geosql-bulk.json

@@ -0,0 +1,33 @@
+{"index":{"_id": "1"}}
+{"region": "Americas", "city": "Mountain View", "location": "37.386483,-122.083843"}
+{"index":{"_id": "2"}}
+{"region": "Americas", "city": "Chicago", "location": "41.888783,-87.637874"}
+{"index":{"_id": "3"}}
+{"region": "Americas", "city": "New York", "location": "40.745171,-73.990027"}
+{"index":{"_id": "4"}}
+{"region": "Americas", "city": "San Francisco", "location": "37.789541,-122.394228"}
+{"index":{"_id": "5"}}
+{"region": "Americas", "city": "Phoenix", "location": "33.376242,-111.973505"}
+{"index":{"_id": "6"}}
+{"region": "Europe", "city": "Amsterdam", "location": "52.347557,4.850312"}
+{"index":{"_id": "7"}}
+{"region": "Europe", "city": "Berlin", "location": "52.486701,13.390889"}
+{"index":{"_id": "8"}}
+{"region": "Europe", "city": "Munich", "location": "48.146321,11.537505"}
+{"index":{"_id": "9"}}
+{"region": "Europe", "city": "London", "location": "51.510871,-0.121672"}
+{"index":{"_id": "10"}}
+{"region": "Europe", "city": "Paris", "location": "48.845538,2.351773"}
+{"index":{"_id": "11"}}
+{"region": "Asia", "city": "Singapore", "location": "1.295868,103.855535"}
+{"index":{"_id": "12"}}
+{"region": "Asia", "city": "Hong Kong", "location": "22.281397,114.183925"}
+{"index":{"_id": "13"}}
+{"region": "Asia", "city": "Seoul", "location": "37.509132,127.060851"}
+{"index":{"_id": "14"}}
+{"region": "Asia", "city": "Tokyo", "location": "35.669616,139.76402225"}
+{"index":{"_id": "15"}}
+{"region": "Asia", "city": "Sydney", "location": "-33.863385,151.208629"}
+
+
+

x-pack/qa/sql/src/main/resources/geo/geosql.csv-spec

@@ -0,0 +1,45 @@
+//
+// Commands on geo test data
+//
+
+showTables
+SHOW TABLES 'geo';
+
+ name:s        | type:s
+geo       |BASE TABLE
+;
+
+// DESCRIBE
+
+describe
+DESCRIBE "geo";
+
+       column:s      |   type:s
+city                 | VARCHAR
+location             | OTHER
+region               | VARCHAR
+;
+
+// SELECT ALL
+// TODO: For now we just get geopoint formatted as is and we also need to convert it to STRING to work with CSV
+
+selectAllPoints
+SELECT city, CAST(location AS STRING) location, region FROM "geo" ORDER BY "city";
+
+     city:s    |     location:s       |  region:s
+Amsterdam      |52.347557,4.850312    |Europe
+Berlin         |52.486701,13.390889   |Europe
+Chicago        |41.888783,-87.637874  |Americas
+Hong Kong      |22.281397,114.183925  |Asia
+London         |51.510871,-0.121672   |Europe
+Mountain View  |37.386483,-122.083843 |Americas
+Munich         |48.146321,11.537505   |Europe
+New York       |40.745171,-73.990027  |Americas
+Paris          |48.845538,2.351773    |Europe
+Phoenix        |33.376242,-111.973505 |Americas
+San Francisco  |37.789541,-122.394228 |Americas
+Seoul          |37.509132,127.060851  |Asia
+Singapore      |1.295868,103.855535   |Asia
+Sydney         |-33.863385,151.208629 |Asia
+Tokyo          |35.669616,139.76402225|Asia
+;

x-pack/qa/sql/src/main/resources/geo/geosql.json

@@ -0,0 +1,20 @@
+{
+  "settings": {
+    "number_of_shards": 1
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "region": {
+          "type": "keyword"
+        },
+        "city": {
+          "type": "keyword"
+        },
+        "location": {
+          "type": "geo_point"
+        }
+      }
+    }
+  }
+}