diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java index 49e583d57d..a27bfd5142 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java @@ -26,6 +26,7 @@ import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import java.util.Arrays; import java.util.Comparator; @@ -118,6 +119,7 @@ public void close() { /** Execute the query and compare with current result */ private TestCaseReport compareWithOtherDb(String sql, DBResult esResult) { + List mismatchResults = Lists.newArrayList(esResult); StringBuilder reasons = new StringBuilder(); for (int i = 0; i < otherDbConnections.length; i++) { try { @@ -126,9 +128,11 @@ private TestCaseReport compareWithOtherDb(String sql, DBResult esResult) { return new SuccessTestCase(nextId(), sql); } + mismatchResults.add(otherDbResult); + // Cannot find any database result match if (i == otherDbConnections.length - 1) { - return new FailedTestCase(nextId(), sql, Arrays.asList(esResult, otherDbResult)); + return new FailedTestCase(nextId(), sql, mismatchResults); } } catch (Exception e) { // Ignore and move on to next database diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java index 96c14798c6..7519a4e0f6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.google.common.base.Strings; import org.json.JSONObject; import java.sql.Connection; @@ -140,7 +141,13 @@ private String getValueList(String[] fieldValues) { private void populateMetaData(ResultSet resultSet, DBResult result) throws SQLException { ResultSetMetaData metaData = resultSet.getMetaData(); for (int i = 1; i <= metaData.getColumnCount(); i++) { - result.addColumn(metaData.getColumnName(i), metaData.getColumnTypeName(i)); + + // Use label name (alias) if present. + String colName = metaData.getColumnLabel(i); + if (Strings.isNullOrEmpty(colName)) { + colName = metaData.getColumnName(i); + } + result.addColumn(colName, metaData.getColumnTypeName(i)); } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java index a40f79e6a1..9c47fa5c19 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/DBResult.java @@ -16,14 +16,16 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import com.google.common.collect.ImmutableSet; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.ToString; import org.json.JSONPropertyName; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -35,11 +37,15 @@ @ToString public class DBResult { + /** Possible types for floating point number */ + private static final Set FLOAT_TYPES = ImmutableSet.of("FLOAT", "DOUBLE", "REAL"); + /** Database name for display */ private final String databaseName; /** Column name and types from result set meta data */ - private final Map colNameAndTypes; + @Getter + private final Collection schema; /** Data rows from result set */ private final Collection dataRows; @@ -49,21 +55,27 @@ public class DBResult { * with specific column names in SELECT but without ORDER BY. */ public DBResult(String databaseName) { - this(databaseName, new LinkedHashMap<>(), new HashSet<>()); + this(databaseName, new ArrayList<>(), new HashSet<>()); } - public DBResult(String databaseName, Map colNameAndTypes, Collection rows) { + public DBResult(String databaseName, Collection schema, Collection rows) { this.databaseName = databaseName; - this.colNameAndTypes = colNameAndTypes; + this.schema = schema; this.dataRows = rows; } public int columnSize() { - return colNameAndTypes.size(); + return schema.size(); } public void addColumn(String name, String type) { - colNameAndTypes.put(StringUtils.toUpper(name), StringUtils.toUpper(type)); + type = StringUtils.toUpper(type); + + // Ignore float type by assigning all type names string to it. + if (FLOAT_TYPES.contains(type)) { + type = FLOAT_TYPES.toString(); + } + schema.add(new Type(StringUtils.toUpper(name), type)); } public void addRow(Row row) { @@ -75,11 +87,6 @@ public String getDatabaseName() { return databaseName; } - @JSONPropertyName("schema") - public Map getColumnNameAndTypes() { - return colNameAndTypes; - } - /** Flatten for simplifying json generated */ public Collection> getDataRows() { return dataRows.stream().map(Row::getValues).collect(Collectors.toSet()); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java index a1cf33d728..3f83d8b4e6 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Row.java @@ -49,7 +49,7 @@ public void add(Object value) { private Object roundFloatNum(Object value) { if (value instanceof Float) { BigDecimal decimal = BigDecimal.valueOf((Float) value).setScale(2, RoundingMode.CEILING); - value = decimal.floatValue(); + value = decimal.doubleValue(); // Convert to double too } else if (value instanceof Double) { BigDecimal decimal = BigDecimal.valueOf((Double) value).setScale(2, RoundingMode.CEILING); value = decimal.doubleValue(); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java new file mode 100644 index 0000000000..de5fd347dd --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/resultset/Type.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset; + +import lombok.Data; + +/** + * Column type in schema + */ +@Data +public class Type { + + /** Column name */ + private final String name; + + /** Column type */ + private final String type; + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java index ee0b202e65..5630b80873 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java @@ -23,8 +23,8 @@ import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.DBConnection; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet; -import com.google.common.collect.ImmutableMap; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -63,10 +63,10 @@ public void setUp() { @Test public void testSuccess() { when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))) + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) ); when(otherDbConnection.select(anyString())).thenReturn( - new DBResult("Other DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))) + new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) ); TestReport expected = new TestReport(); @@ -77,8 +77,8 @@ public void testSuccess() { @Test public void testFailureDueToInconsistency() { - DBResult esResult = new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("JOHN")))); + DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); when(esConnection.select(anyString())).thenReturn(esResult); when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); @@ -96,9 +96,9 @@ public void testSuccessFinally() { esConnection, new DBConnection[]{otherDbConnection, anotherDbConnection} ); - DBResult esResult = new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("JOHN")))); - DBResult anotherDbResult = new DBResult("Another DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))); + DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); + DBResult anotherDbResult = new DBResult("Another DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); when(esConnection.select(anyString())).thenReturn(esResult); when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); @@ -117,15 +117,15 @@ public void testFailureDueToEventualInconsistency() { esConnection, new DBConnection[]{otherDbConnection, anotherDbConnection} ); - DBResult esResult = new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))); - DBResult otherDbResult = new DBResult("Other DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("JOHN")))); - DBResult anotherDbResult = new DBResult("ZZZ DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("Hank")))); + DBResult esResult = new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))); + DBResult otherDbResult = new DBResult("Other DB", asList(new Type("firstname", "text")), asList(new Row(asList("JOHN")))); + DBResult anotherDbResult = new DBResult("ZZZ DB", asList(new Type("firstname", "text")), asList(new Row(asList("Hank")))); when(esConnection.select(anyString())).thenReturn(esResult); when(otherDbConnection.select(anyString())).thenReturn(otherDbResult); when(anotherDbConnection.select(anyString())).thenReturn(anotherDbResult); TestReport expected = new TestReport(); - expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, anotherDbResult))); + expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult, anotherDbResult))); TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts")); assertEquals(expected, actual); } @@ -143,7 +143,7 @@ public void testErrorDueToESException() { @Test public void testErrorDueToNoOtherDBSupportThisQuery() { when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))) + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) ); when(otherDbConnection.select(anyString())).thenThrow(new RuntimeException("Unsupported feature")); @@ -162,11 +162,11 @@ public void testSuccessWhenOneDBSupportThisQuery() { ); when(esConnection.select(anyString())).thenReturn( - new DBResult("ES", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))) + new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) ); when(otherDbConnection.select(anyString())).thenThrow(new RuntimeException("Unsupported feature")); when(anotherDbConnection.select(anyString())).thenReturn( - new DBResult("Another DB", ImmutableMap.of("firstname", "text"), asList(new Row(asList("John")))) + new DBResult("Another DB", asList(new Type("firstname", "text")), asList(new Row(asList("John")))) ); TestReport expected = new TestReport(); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java index d3c16b7a27..c71f996599 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/DBResultTest.java @@ -16,9 +16,11 @@ package com.amazon.opendistroforelasticsearch.sql.correctness.tests; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.google.common.collect.ImmutableMap; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; import org.junit.Test; +import java.util.Arrays; + import static java.util.Collections.emptyList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -30,22 +32,22 @@ public class DBResultTest { @Test public void dbResultFromDifferentDbNameShouldEqual() { - DBResult result1 = new DBResult("DB 1", ImmutableMap.of("name", "VARCHAR"), emptyList()); - DBResult result2 = new DBResult("DB 2", ImmutableMap.of("name", "VARCHAR"), emptyList()); + DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); + DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); assertEquals(result1, result2); } @Test public void dbResultWithDifferentColumnShouldNotEqual() { - DBResult result1 = new DBResult("DB 1", ImmutableMap.of("name", "VARCHAR"), emptyList()); - DBResult result2 = new DBResult("DB 2", ImmutableMap.of("age", "INT"), emptyList()); + DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("name", "VARCHAR")), emptyList()); + DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); assertNotEquals(result1, result2); } @Test public void dbResultWithDifferentColumnTypeShouldNotEqual() { - DBResult result1 = new DBResult("DB 1", ImmutableMap.of("age", "FLOAT"), emptyList()); - DBResult result2 = new DBResult("DB 2", ImmutableMap.of("age", "INT"), emptyList()); + DBResult result1 = new DBResult("DB 1", Arrays.asList(new Type("age", "FLOAT")), emptyList()); + DBResult result2 = new DBResult("DB 2", Arrays.asList(new Type("age", "INT")), emptyList()); assertNotEquals(result1, result2); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java index b039bd94a1..bb2cfd32bf 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/JDBCConnectionTest.java @@ -17,7 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.correctness.runner.connection.JDBCConnection; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; -import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.junit.Before; @@ -116,8 +116,11 @@ public void testSelectQuery() throws SQLException { DBResult result = conn.select("SELECT * FROM test"); assertEquals("Test DB", result.getDatabaseName()); assertEquals( - ImmutableMap.of("NAME", "VARCHAR", "AGE", "INT"), - result.getColumnNameAndTypes() + Arrays.asList( + new Type("NAME", "VARCHAR"), + new Type("AGE", "INT") + ), + result.getSchema() ); assertEquals( Sets.newHashSet( @@ -128,6 +131,23 @@ public void testSelectQuery() throws SQLException { ); } + @Test + public void testSelectQueryWithAlias() throws SQLException { + ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "age", "INT"), "n", "a"); + ResultSet resultSet = mockResultSet(new Object[]{"John", 25}, new Object[]{"Hank", 30}); + when(statement.executeQuery(anyString())).thenReturn(resultSet); + when(resultSet.getMetaData()).thenReturn(metaData); + + DBResult result = conn.select("SELECT * FROM test"); + assertEquals( + Arrays.asList( + new Type("N", "VARCHAR"), + new Type("A", "INT") + ), + result.getSchema() + ); + } + @Test public void testSelectQueryWithFloatInResultSet() throws SQLException { ResultSetMetaData metaData = mockMetaData(ImmutableMap.of("name", "VARCHAR", "balance", "FLOAT")); @@ -140,6 +160,13 @@ public void testSelectQueryWithFloatInResultSet() throws SQLException { when(resultSet.getMetaData()).thenReturn(metaData); DBResult result = conn.select("SELECT * FROM test"); + assertEquals( + Arrays.asList( + new Type("NAME", "VARCHAR"), + new Type("BALANCE", "[FLOAT, DOUBLE, REAL]") + ), + result.getSchema() + ); assertEquals( Sets.newHashSet( Arrays.asList("John", 25.13), @@ -167,7 +194,7 @@ private ResultSet mockResultSet(Object[]... rows) throws SQLException { return resultSet; } - private ResultSetMetaData mockMetaData(Map nameAndTypes) throws SQLException { + private ResultSetMetaData mockMetaData(Map nameAndTypes, String... aliases) throws SQLException { ResultSetMetaData metaData = mock(ResultSetMetaData.class); OngoingStubbing getColumnName = when(metaData.getColumnName(anyInt())); @@ -180,6 +207,13 @@ private ResultSetMetaData mockMetaData(Map nameAndTypes) throws getColumnTypeName = getColumnTypeName.thenReturn(value); } + if (aliases.length > 0) { + OngoingStubbing getColumnLabel = when(metaData.getColumnLabel(anyInt())); + for (String alias : aliases) { + getColumnLabel = getColumnLabel.thenReturn(alias); + } + } + when(metaData.getColumnCount()).thenReturn(nameAndTypes.size()); return metaData; } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java index 4bf917e25b..7d15899d23 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java @@ -21,7 +21,7 @@ import com.amazon.opendistroforelasticsearch.sql.correctness.report.TestReport; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.DBResult; import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row; -import com.google.common.collect.ImmutableMap; +import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type; import org.json.JSONObject; import org.junit.Test; @@ -65,8 +65,8 @@ public void testSuccessReport() { @Test public void testFailedReport() { report.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts", asList( - new DBResult("Elasticsearch", ImmutableMap.of("firstName", "text"), singleton(new Row(asList("hello")))), - new DBResult("H2", ImmutableMap.of("firstName", "text"), singleton(new Row(asList("world")))) + new DBResult("Elasticsearch", singleton(new Type("firstName", "text")), singleton(new Row(asList("hello")))), + new DBResult("H2", singleton(new Type("firstName", "text")), singleton(new Row(asList("world")))) ))); JSONObject actual = new JSONObject(report); JSONObject expected = new JSONObject( @@ -84,16 +84,22 @@ public void testFailedReport() { " \"resultSets\": [" + " {" + " \"database\": \"Elasticsearch\"," + - " \"schema\": {" + - " \"firstName\": \"text\"" + - " }," + + " \"schema\": [" + + " {" + + " \"name\": \"firstName\"," + + " \"type\": \"text\"" + + " }" + + " ]," + " \"dataRows\": [[\"hello\"]]" + " }," + " {" + " \"database\": \"H2\"," + - " \"schema\": {" + - " \"firstName\": \"text\"" + - " }," + + " \"schema\": [" + + " {" + + " \"name\": \"firstName\"," + + " \"type\": \"text\"" + + " }" + + " ]," + " \"dataRows\": [[\"world\"]]" + " }" + " ]" + diff --git a/src/test/resources/correctness/kibana_sample_data_flights.csv b/src/test/resources/correctness/kibana_sample_data_flights.csv index b3ffb73e23..94efedcafb 100644 --- a/src/test/resources/correctness/kibana_sample_data_flights.csv +++ b/src/test/resources/correctness/kibana_sample_data_flights.csv @@ -1,21 +1,21 @@ FlightNum,Origin,FlightDelay,DistanceMiles,FlightTimeMin,OriginWeather,dayOfWeek,AvgTicketPrice,Carrier,FlightDelayMin,OriginRegion,FlightDelayType,DestAirportID,Dest,FlightTimeHour,Cancelled,DistanceKilometers,OriginCityName,DestWeather,OriginCountry,DestCountry,DestRegion,OriginAirportID,DestCityName,timestamp -RGXY9H5,Chubu Centrair International Airport,false,1619.970725161303,124.1471507959044,Heavy Fog,0,626.1297405910661,Kibana Airlines,0,SE-BD,No Delay,CAN,Guangzhou Baiyun International Airport,2.06911917993174,true,2607.0901667139924,Tokoname,Clear,JP,CN,SE-BD,NGO,Guangzhou,2019-12-23T11:19:32 -WOPNZEP,Munich Airport,true,198.57903689856937,34.9738738474057,Sunny,0,681.9911763989377,Kibana Airlines,15,DE-BY,Carrier Delay,VE05,Venice Marco Polo Airport,0.5828978974567617,false,319.58198155849124,Munich,Cloudy,DE,IT,IT-34,MUC,Venice,2019-12-23T12:32:26 -G9J5O2V,Frankfurt am Main Airport,false,4857.154739888458,651.402736475921,Clear,0,868.0507463122127,Kibana Airlines,0,DE-HE,No Delay,XIY,Xi'an Xianyang International Airport,10.856712274598683,false,7816.832837711051,Frankfurt am Main,Thunder & Lightning,DE,CN,SE-BD,FRA,Xi'an,2019-12-23T03:48:33 -HM80A5V,Itami Airport,false,5862.6666599206,555.0027890084269,Heavy Fog,0,765.0413127727119,Logstash Airways,0,SE-BD,No Delay,TV01,Treviso-Sant'Angelo Airport,9.250046483473783,true,9435.047413143258,Osaka,Clear,JP,IT,IT-34,ITM,Treviso,2019-12-23T19:50:48 -84B0Y32,Charles de Gaulle International Airport,false,4397.926660603864,372.51457282541395,Thunder & Lightning,0,913.1638984616233,Kibana Airlines,0,FR-J,No Delay,STL,St Louis Lambert International Airport,6.208576213756899,false,7077.776883682865,Paris,Thunder & Lightning,FR,US,US-MO,CDG,St Louis,2019-12-23T11:30:48 -2AZWPJX,Rajiv Gandhi International Airport,true,0,180,Sunny,0,103.25307304704197,Logstash Airways,180,SE-BD,Security Delay,HYD,Rajiv Gandhi International Airport,3,false,0,Hyderabad,Hail,IN,IN,SE-BD,HYD,Hyderabad,2019-12-23T19:52:54 -SFLRI9O,Erie International Tom Ridge Field,false,6961.655654280931,622.4277087379495,Clear,0,775.1109173747694,Kibana Airlines,0,US-PA,No Delay,CJU,Jeju International Airport,10.373795145632492,false,11203.698757283091,Erie,Clear,US,KR,SE-BD,ERI,Jeju City,2019-12-23T07:32:32 -QDQMOD6,Brisbane International Airport,false,8013.330880747018,716.4558873858294,Thunder & Lightning,0,832.082612870741,ES-Air,0,SE-BD,No Delay,DEN,Denver International Airport,11.94093145643049,false,12896.20597294493,Brisbane,Cloudy,AU,US,US-CO,BNE,Denver,2019-12-23T10:59:26 -XTGFN9A,Jorge Chavez International Airport,false,3946.924514217792,396.99745533808243,Thunder & Lightning,0,348.23579123315324,Kibana Airlines,0,SE-BD,No Delay,YOW,Ottawa Macdonald-Cartier International Airport,6.616624255634707,false,6351.959285409319,Lima,Rain,PE,CA,CA-ON,LIM,Ottawa,2019-12-23T21:10:09 -USRQ3KK,Stockholm-Arlanda Airport,false,996.8381561540818,94.36797091633146,Clear,0,661.3465606549407,ES-Air,0,SE-AB,No Delay,TV01,Treviso-Sant'Angelo Airport,1.572799515272191,false,1604.2555055776347,Stockholm,Clear,SE,IT,IT-34,ARN,Treviso,2019-12-23T04:33:56 -PK46NHH,Milano Linate Airport,false,5261.396351845886,604.8140464617903,Rain,0,600.4401843290168,JetBeats,0,IT-25,No Delay,GEG,Spokane International Airport,10.080234107696505,false,8467.396650465065,Milan,Clear,IT,US,US-WA,MI11,Spokane,2019-12-23T20:35:25 -G80VHCJ,Bari Karol Wojty__a Airport,false,5630.111629019724,604.0524246328747,Sunny,0,738.260189539631,Logstash Airways,0,IT-75,No Delay,CJU,Jeju International Airport,10.067540410547911,false,9060.78636949312,Bari,Rain,IT,KR,SE-BD,BA02,Jeju City,2019-12-23T10:59:56 -PDS4U17,El Dorado International Airport,false,5591.079567130033,499.887241937962,Thunder & Lightning,0,437.9253204442997,ES-Air,0,SE-BD,No Delay,TO11,Turin Airport,8.331454032299368,false,8997.970354883317,Bogota,Hail,CO,IT,IT-21,BOG,Torino,2019-12-23T10:33:53 -2MXRGRK,Abu Dhabi International Airport,false,8160.7690090650885,656.6742320062424,Cloudy,0,825.9174161826418,JetBeats,0,SE-BD,No Delay,ABQ,Albuquerque International Sunport Airport,10.944570533437373,false,13133.484640124847,Abu Dhabi,Thunder & Lightning,AE,US,US-NM,AUH,Albuquerque,2019-12-23T19:27:11 -57CZEDA,London Heathrow Airport,true,4757.876231054233,720.4152685405732,Damaging Wind,0,836.1010286937247,ES-Air,270,GB-ENG,Carrier Delay,XHBU,Ukrainka Air Base,12.006921142342886,false,7657.059565189745,London,Sunny,GB,RU,RU-AMU,LHR,Belogorsk,2019-12-23T18:48:49 -5FYALP0,Malpensa International Airport,false,5812.230334559898,492.30936923905085,Damaging Wind,0,417.34744554513884,JetBeats,0,IT-25,No Delay,LAS,McCarran International Airport,8.20515615398418,false,9353.878015541966,Milan,Clear,IT,US,US-NV,MI12,Las Vegas,2019-12-23T10:37:54 -HVWAL6J,Comodoro Arturo Merino Benitez International Airport,false,7292.7292896018525,617.7110592550002,Cloudy,0,946.888426456834,Logstash Airways,0,SE-BD,No Delay,PA03,Falcone Borsellino Airport,10.29518432091667,false,11736.510125845005,Santiago,Cloudy,CL,IT,IT-82,SCL,Palermo,2019-12-23T03:54:12 -7ORM12S,Leonardo da Vinci___Fiumicino Airport,false,160.39074208529965,23.46580713004768,Sunny,0,118.37483602607261,Kibana Airlines,0,IT-62,No Delay,PI05,Pisa International Airport,0.39109678550079463,false,258.1238784305245,Rome,Sunny,IT,IT,IT-52,RM11,Pisa,2019-12-23T03:54:12 -2P36OEP,New Chitose Airport,true,5340.290617241973,941.1970552595557,Cloudy,0,705.7149863531135,Kibana Airlines,225,SE-BD,Late Aircraft Delay,VIE,Vienna International Airport,15.686617587659262,false,8594.364663114668,Chitose / Tomakomai,Rain,JP,AT,AT-9,CTS,Vienna,2019-12-23T09:41:52 -HLNZHCX,Verona Villafranca Airport,false,0,0,Sunny,0,172.3790782673846,ES-Air,0,IT-34,No Delay,VR10,Verona Villafranca Airport,0,false,0,Verona,Sunny,IT,IT,IT-34,VR10,Verona,2019-12-23T19:34:51 +RGXY9H5,Chubu Centrair International Airport,false,1619.970725161303,124.1471507959044,Heavy Fog,0,626.1297405910661,Kibana Airlines,0,SE-BD,No Delay,CAN,Guangzhou Baiyun International Airport,2.06911917993174,true,2607.0901667139924,Tokoname,Clear,JP,CN,SE-BD,NGO,Guangzhou,2019-12-23 11:19:32 +WOPNZEP,Munich Airport,true,198.57903689856937,34.9738738474057,Sunny,0,681.9911763989377,Kibana Airlines,15,DE-BY,Carrier Delay,VE05,Venice Marco Polo Airport,0.5828978974567617,false,319.58198155849124,Munich,Cloudy,DE,IT,IT-34,MUC,Venice,2019-12-23 12:32:26 +G9J5O2V,Frankfurt am Main Airport,false,4857.154739888458,651.402736475921,Clear,0,868.0507463122127,Kibana Airlines,0,DE-HE,No Delay,XIY,Xi'an Xianyang International Airport,10.856712274598683,false,7816.832837711051,Frankfurt am Main,Thunder & Lightning,DE,CN,SE-BD,FRA,Xi'an,2019-12-23 03:48:33 +HM80A5V,Itami Airport,false,5862.6666599206,555.0027890084269,Heavy Fog,0,765.0413127727119,Logstash Airways,0,SE-BD,No Delay,TV01,Treviso-Sant'Angelo Airport,9.250046483473783,true,9435.047413143258,Osaka,Clear,JP,IT,IT-34,ITM,Treviso,2019-12-23 19:50:48 +84B0Y32,Charles de Gaulle International Airport,false,4397.926660603864,372.51457282541395,Thunder & Lightning,0,913.1638984616233,Kibana Airlines,0,FR-J,No Delay,STL,St Louis Lambert International Airport,6.208576213756899,false,7077.776883682865,Paris,Thunder & Lightning,FR,US,US-MO,CDG,St Louis,2019-12-23 11:30:48 +2AZWPJX,Rajiv Gandhi International Airport,true,0,180,Sunny,0,103.25307304704197,Logstash Airways,180,SE-BD,Security Delay,HYD,Rajiv Gandhi International Airport,3,false,0,Hyderabad,Hail,IN,IN,SE-BD,HYD,Hyderabad,2019-12-23 19:52:54 +SFLRI9O,Erie International Tom Ridge Field,false,6961.655654280931,622.4277087379495,Clear,0,775.1109173747694,Kibana Airlines,0,US-PA,No Delay,CJU,Jeju International Airport,10.373795145632492,false,11203.698757283091,Erie,Clear,US,KR,SE-BD,ERI,Jeju City,2019-12-23 07:32:32 +QDQMOD6,Brisbane International Airport,false,8013.330880747018,716.4558873858294,Thunder & Lightning,0,832.082612870741,ES-Air,0,SE-BD,No Delay,DEN,Denver International Airport,11.94093145643049,false,12896.20597294493,Brisbane,Cloudy,AU,US,US-CO,BNE,Denver,2019-12-23 10:59:26 +XTGFN9A,Jorge Chavez International Airport,false,3946.924514217792,396.99745533808243,Thunder & Lightning,0,348.23579123315324,Kibana Airlines,0,SE-BD,No Delay,YOW,Ottawa Macdonald-Cartier International Airport,6.616624255634707,false,6351.959285409319,Lima,Rain,PE,CA,CA-ON,LIM,Ottawa,2019-12-23 21:10:09 +USRQ3KK,Stockholm-Arlanda Airport,false,996.8381561540818,94.36797091633146,Clear,0,661.3465606549407,ES-Air,0,SE-AB,No Delay,TV01,Treviso-Sant'Angelo Airport,1.572799515272191,false,1604.2555055776347,Stockholm,Clear,SE,IT,IT-34,ARN,Treviso,2019-12-23 04:33:56 +PK46NHH,Milano Linate Airport,false,5261.396351845886,604.8140464617903,Rain,0,600.4401843290168,JetBeats,0,IT-25,No Delay,GEG,Spokane International Airport,10.080234107696505,false,8467.396650465065,Milan,Clear,IT,US,US-WA,MI11,Spokane,2019-12-23 20:35:25 +G80VHCJ,Bari Karol Wojty__a Airport,false,5630.111629019724,604.0524246328747,Sunny,0,738.260189539631,Logstash Airways,0,IT-75,No Delay,CJU,Jeju International Airport,10.067540410547911,false,9060.78636949312,Bari,Rain,IT,KR,SE-BD,BA02,Jeju City,2019-12-23 10:59:56 +PDS4U17,El Dorado International Airport,false,5591.079567130033,499.887241937962,Thunder & Lightning,0,437.9253204442997,ES-Air,0,SE-BD,No Delay,TO11,Turin Airport,8.331454032299368,false,8997.970354883317,Bogota,Hail,CO,IT,IT-21,BOG,Torino,2019-12-23 10:33:53 +2MXRGRK,Abu Dhabi International Airport,false,8160.7690090650885,656.6742320062424,Cloudy,0,825.9174161826418,JetBeats,0,SE-BD,No Delay,ABQ,Albuquerque International Sunport Airport,10.944570533437373,false,13133.484640124847,Abu Dhabi,Thunder & Lightning,AE,US,US-NM,AUH,Albuquerque,2019-12-23 19:27:11 +57CZEDA,London Heathrow Airport,true,4757.876231054233,720.4152685405732,Damaging Wind,0,836.1010286937247,ES-Air,270,GB-ENG,Carrier Delay,XHBU,Ukrainka Air Base,12.006921142342886,false,7657.059565189745,London,Sunny,GB,RU,RU-AMU,LHR,Belogorsk,2019-12-23 18:48:49 +5FYALP0,Malpensa International Airport,false,5812.230334559898,492.30936923905085,Damaging Wind,0,417.34744554513884,JetBeats,0,IT-25,No Delay,LAS,McCarran International Airport,8.20515615398418,false,9353.878015541966,Milan,Clear,IT,US,US-NV,MI12,Las Vegas,2019-12-23 10:37:54 +HVWAL6J,Comodoro Arturo Merino Benitez International Airport,false,7292.7292896018525,617.7110592550002,Cloudy,0,946.888426456834,Logstash Airways,0,SE-BD,No Delay,PA03,Falcone Borsellino Airport,10.29518432091667,false,11736.510125845005,Santiago,Cloudy,CL,IT,IT-82,SCL,Palermo,2019-12-23 03:54:12 +7ORM12S,Leonardo da Vinci___Fiumicino Airport,false,160.39074208529965,23.46580713004768,Sunny,0,118.37483602607261,Kibana Airlines,0,IT-62,No Delay,PI05,Pisa International Airport,0.39109678550079463,false,258.1238784305245,Rome,Sunny,IT,IT,IT-52,RM11,Pisa,2019-12-23 03:54:12 +2P36OEP,New Chitose Airport,true,5340.290617241973,941.1970552595557,Cloudy,0,705.7149863531135,Kibana Airlines,225,SE-BD,Late Aircraft Delay,VIE,Vienna International Airport,15.686617587659262,false,8594.364663114668,Chitose / Tomakomai,Rain,JP,AT,AT-9,CTS,Vienna,2019-12-23 09:41:52 +HLNZHCX,Verona Villafranca Airport,false,0,0,Sunny,0,172.3790782673846,ES-Air,0,IT-34,No Delay,VR10,Verona Villafranca Airport,0,false,0,Verona,Sunny,IT,IT,IT-34,VR10,Verona,2019-12-23 19:34:51 diff --git a/src/test/resources/correctness/kibana_sample_data_flights.json b/src/test/resources/correctness/kibana_sample_data_flights.json index ca6030983a..d180b6283c 100644 --- a/src/test/resources/correctness/kibana_sample_data_flights.json +++ b/src/test/resources/correctness/kibana_sample_data_flights.json @@ -74,7 +74,8 @@ "type": "integer" }, "timestamp" : { - "type" : "date" + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" } } } diff --git a/src/test/resources/correctness/sanity_integration_tests.txt b/src/test/resources/correctness/sanity_integration_tests.txt index 82cd2f8471..11aed79de6 100644 --- a/src/test/resources/correctness/sanity_integration_tests.txt +++ b/src/test/resources/correctness/sanity_integration_tests.txt @@ -1,8 +1,51 @@ SELECT AvgTicketPrice, Cancelled, Carrier, FlightDelayMin, timestamp FROM kibana_sample_data_flights SELECT AvgTicketPrice AS avg, Cancelled AS cancel, Carrier AS carrier, FlightDelayMin AS delay, timestamp AS ts FROM kibana_sample_data_flights SELECT f.AvgTicketPrice AS avg, f.Cancelled AS cancel, f.Carrier AS carrier, f.FlightDelayMin AS delay, timestamp AS ts FROM kibana_sample_data_flights f -SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier -SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier HAVING AVG(FlightDelayMin) > 5 +SELECT LOG(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT LOG2(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT LOG10(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT LN(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT SQRT(DistanceMiles) FROM kibana_sample_data_flights +SELECT CBRT(DistanceMiles) FROM kibana_sample_data_flights +SELECT CEIL(FlightTimeMin) FROM kibana_sample_data_flights +SELECT FLOOR(FlightTimeMin) FROM kibana_sample_data_flights +SELECT SIN(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT COS(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT TAN(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT ASIN(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT ACOS(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT ATAN(AvgTicketPrice) FROM kibana_sample_data_flights +SELECT LEFT(Origin, 3) FROM kibana_sample_data_flights +SELECT RIGHT(Origin, 3) FROM kibana_sample_data_flights +SELECT LENGTH(Origin) FROM kibana_sample_data_flights +SELECT ASCII(Origin) FROM kibana_sample_data_flights +SELECT TRIM(Origin) FROM kibana_sample_data_flights +SELECT LOWER(Origin) FROM kibana_sample_data_flights +SELECT UPPER(Origin) FROM kibana_sample_data_flights +SELECT SUBSTRING(Origin, 1, 5) FROM kibana_sample_data_flights SELECT YEAR(timestamp) FROM kibana_sample_data_flights +SELECT MONTH(timestamp) FROM kibana_sample_data_flights +SELECT DAYOFMONTH(timestamp) FROM kibana_sample_data_flights +SELECT MONTH_OF_YEAR(timestamp) FROM kibana_sample_data_flights +SELECT WEEK_OF_YEAR(timestamp) FROM kibana_sample_data_flights +SELECT DAY_OF_YEAR(timestamp) FROM kibana_sample_data_flights +SELECT DAY_OF_MONTH(timestamp) FROM kibana_sample_data_flights +SELECT DAY_OF_WEEK(timestamp) FROM kibana_sample_data_flights +SELECT HOUR_OF_DAY(timestamp) FROM kibana_sample_data_flights +SELECT MINUTE_OF_DAY(timestamp) FROM kibana_sample_data_flights +SELECT MINUTE_OF_HOUR(timestamp) FROM kibana_sample_data_flights +SELECT SECOND_OF_MINUTE(timestamp) FROM kibana_sample_data_flights +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 AND Cancelled = False +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE timestamp > '2019-12-23 10:00:00' +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE kibana_sample_data_flights.Carrier LIKE 'Wind' +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 ORDER BY FlightDelayMin LIMIT 5 +SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 ORDER BY DestAirportID LIMIT 5 +SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier +SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY 1 +SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights WHERE FlightDelay = True GROUP BY Carrier +SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier HAVING AVG(FlightTimeMin) > 100 +SELECT Carrier, MAX(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier HAVING AVG(FlightTimeMin) > 100 +SELECT Carrier, AVG(FlightDelayMin) FROM kibana_sample_data_flights GROUP BY Carrier ORDER BY Carrier LIMIT 2 SELECT AvgTicketPrice AS AvgTicketPrice, DestCountry AS DestCountry FROM kibana_sample_data_ecommerce e JOIN kibana_sample_data_flights f ON (e.day_of_week_i = f.dayOfWeek) LIMIT 1000 SELECT AvgTicketPrice AS AvgTicketPrice, DestCountry AS DestCountry FROM kibana_sample_data_ecommerce e LEFT JOIN kibana_sample_data_flights f ON (e.day_of_week_i = f.dayOfWeek) LIMIT 1000