Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SQL: Fix esType for DATETIME/DATE and INTERVALS #38179

Merged
merged 6 commits into from
Feb 5, 2019
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ else if (DataTypes.isUnsupported(fa.dataType())) {
// compound fields
else if (allowCompound == false && fa.dataType().isPrimitive() == false) {
named = u.withUnresolvedMessage(
"Cannot use field [" + fa.name() + "] type [" + fa.dataType().esType + "] only its subfields");
"Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName + "] only its subfields");
}
}
return named;
Expand Down Expand Up @@ -1228,4 +1228,4 @@ protected boolean skipResolved() {
return true;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ private static void validateInExpression(LogicalPlan p, Set<Failure> localFailur
for (Expression value : in.list()) {
if (areTypesCompatible(dt, value.dataType()) == false) {
localFailures.add(fail(value, "expected data type [{}], value provided is of type [{}]",
dt.esType, value.dataType().esType));
dt.typeName, value.dataType().typeName));
return;
}
}
Expand All @@ -703,7 +703,7 @@ private static void validateConditional(LogicalPlan p, Set<Failure> localFailure
} else {
if (areTypesCompatible(dt, child.dataType()) == false) {
localFailures.add(fail(child, "expected data type [{}], value provided is of type [{}]",
dt.esType, child.dataType().esType));
dt.typeName, child.dataType().typeName));
return;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,4 +175,4 @@ private static void disableSource(SearchSourceBuilder builder) {
builder.storedFields(NO_STORED_FIELD);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public String getWriteableName() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(fieldName);
out.writeOptionalString(dataType == null ? null : dataType.esType);
out.writeOptionalString(dataType == null ? null : dataType.typeName);
out.writeBoolean(useDocValue);
out.writeOptionalString(hitName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public static TypeResolution typeMustBe(Expression e,
paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
acceptedTypesForErrorMsg(acceptedTypes),
Expressions.name(e),
e.dataType().esType));
e.dataType().typeName));
}

private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ protected TypeResolution resolveType() {
((FieldAttribute) field()).exactAttribute();
} catch (MappingException ex) {
return new TypeResolution(format(null, "[{}] cannot operate on first argument field of data type [{}]",
functionName(), field().dataType().esType));
functionName(), field().dataType().typeName));
}

if (orderField() != null) {
Expand All @@ -59,7 +59,7 @@ protected TypeResolution resolveType() {
((FieldAttribute) orderField()).exactAttribute();
} catch (MappingException ex) {
return new TypeResolution(format(null, "[{}] cannot operate on second argument field of data type [{}]",
functionName(), orderField().dataType().esType));
functionName(), orderField().dataType().typeName));
}
}
return TypeResolution.TYPE_RESOLVED;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@

import org.elasticsearch.xpack.sql.expression.Expression;
import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.type.DataTypes;

import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
Expand Down Expand Up @@ -36,7 +36,7 @@ protected Sub replaceChildren(Expression newLeft, Expression newRight) {
protected TypeResolution resolveWithIntervals() {
if (right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) {
return new TypeResolution(format(null, "Cannot subtract a {}[{}] from an interval[{}]; do you mean the reverse?",
right().dataType().esType, right().source().text(), left().source().text()));
right().dataType().typeName, right().source().text(), left().source().text()));
}
return TypeResolution.TYPE_RESOLVED;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ private void fillInRows(Map<String, EsField> mapping, String prefix, List<List<?
String name = e.getKey();
if (dt != null) {
// show only fields that exist in ES
rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName(), dt.esType));
rows.add(asList(prefix != null ? prefix + "." + name : name, dt.sqlName(), dt.typeName));
if (field.getProperties().isEmpty() == false) {
String newPrefix = prefix != null ? prefix + "." + name : name;
fillInRows(field.getProperties(), newPrefix, rows);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.sql.DatabaseMetaData;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -143,7 +142,7 @@ static void fillInRows(String clusterName, String indexName, Map<String, EsField
indexName,
name,
odbcCompatible(type.sqlType.getVendorTypeNumber(), isOdbcClient),
type.esType.toUpperCase(Locale.ROOT),
type.toString(),
type.displaySize,
// TODO: is the buffer_length correct?
type.size,
Expand Down Expand Up @@ -208,4 +207,4 @@ public boolean equals(Object obj) {
&& Objects.equals(pattern, other.pattern)
&& Objects.equals(columnPattern, other.columnPattern);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,14 @@
import org.elasticsearch.xpack.sql.session.Rows;
import org.elasticsearch.xpack.sql.session.SchemaRowSet;
import org.elasticsearch.xpack.sql.session.SqlSession;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.tree.Source;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.DataTypes;

import java.sql.DatabaseMetaData;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.stream.Stream;

import static java.util.Arrays.asList;
Expand Down Expand Up @@ -81,7 +80,7 @@ public final void execute(SqlSession session, ActionListener<SchemaRowSet> liste
List<List<?>> rows = values
// sort by SQL int type (that's what the JDBC/ODBC specs want) followed by name
.sorted(Comparator.comparing((DataType t) -> t.sqlType.getVendorTypeNumber()).thenComparing(DataType::sqlName))
.map(t -> asList(t.esType.toUpperCase(Locale.ROOT),
.map(t -> asList(t.toString(),
t.sqlType.getVendorTypeNumber(),
//https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017
t.defaultPrecision,
Expand Down Expand Up @@ -132,4 +131,4 @@ public boolean equals(Object obj) {

return type.equals(((SysTypes) obj).type);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,9 @@ static SqlQueryResponse createResponse(SqlQueryRequest request, SchemaRowSet row
List<ColumnInfo> columns = new ArrayList<>(rowSet.columnCount());
for (Schema.Entry entry : rowSet.schema()) {
if (Mode.isDriver(request.mode())) {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType, entry.type().displaySize));
columns.add(new ColumnInfo("", entry.name(), entry.type().typeName, entry.type().displaySize));
} else {
columns.add(new ColumnInfo("", entry.name(), entry.type().esType));
columns.add(new ColumnInfo("", entry.name(), entry.type().typeName));
}
}
columns = unmodifiableList(columns);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,6 @@ public DataType fieldDataType() {

@Override
public String toString() {
return ">" + name + "[" + fieldDataType.esType + "]";
return ">" + name + "[" + fieldDataType.typeName + "]";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,12 @@ public enum DataType {


/**
* Elasticsearch type name
* Type's name used for error messages and column info for the clients
*/
public final String typeName;

/**
* Elasticsearch data type that it maps to
*/
public final String esType;

Expand Down Expand Up @@ -176,7 +181,8 @@ public enum DataType {

DataType(SQLType sqlType, int size, int defaultPrecision, int displaySize, boolean isInteger,
boolean isRational, boolean defaultDocValues) {
this.esType = name().toLowerCase(Locale.ROOT);
this.typeName = name().toLowerCase(Locale.ROOT);
this.esType = name().equals("DATETIME") ? "date" : typeName;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think a better way would be to ask this parameter for the constructor and add it for each declaration.
So DATETIME would be "date", INTERVAL_... null while NULL would be "null". Potentially add an overloaded constructor to specify esType name which by default would be the enum name lowercase but with the possibility of being overwritten.

this.sqlType = sqlType;
this.size = size;
this.defaultPrecision = defaultPrecision;
Expand Down Expand Up @@ -228,8 +234,6 @@ public static DataType fromOdbcType(String odbcType) {

/**
* Creates returns DataType enum corresponding to the specified es type
* <p>
* For any dataType DataType.fromTypeName(dataType.esType) == dataType
*/
public static DataType fromTypeName(String esType) {
String uppercase = esType.toUpperCase(Locale.ROOT);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ public String toString() {
}
sb.append(names.get(i));
sb.append(":");
sb.append(types.get(i).esType);
sb.append(types.get(i).typeName);
}
sb.append("]");
return sb.toString();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public static Map<String, Map<String, FieldCapabilities>> fromMappings(EsIndex..
if (entry.getValue().size() > 1) {
for (EsIndex index : indices) {
EsField field = index.mapping().get(fieldName);
UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esType);
UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().typeName);
fieldCaps.indices.add(index.name());
}
//TODO: what about nonAgg/SearchIndices?
Expand All @@ -171,7 +171,7 @@ private static void addFieldCaps(String parent, EsField field, String indexName,
map = new HashMap<>();
merged.put(fieldName, map);
}
FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esType,
FieldCapabilities caps = map.computeIfAbsent(field.getDataType().typeName,
esType -> new UpdateableFieldCapabilities(fieldName, esType,
isSearchable(field.getDataType()),
isAggregatable(field.getDataType())));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class ParameterTests extends ESTestCase {
public void testSingleParameter() {
Expression expression = new SqlParser().createExpression("a = \n?",
Collections.singletonList(
new SqlTypedParamValue(DataType.KEYWORD.esType, "foo")
new SqlTypedParamValue(DataType.KEYWORD.typeName, "foo")
));
logger.info(expression);
assertThat(expression, instanceOf(Equals.class));
Expand All @@ -42,10 +42,10 @@ public void testSingleParameter() {

public void testMultipleParameters() {
Expression expression = new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList(
new SqlTypedParamValue(DataType.LONG.esType, 1L),
new SqlTypedParamValue(DataType.LONG.esType, 2L),
new SqlTypedParamValue(DataType.LONG.esType, 3L),
new SqlTypedParamValue(DataType.LONG.esType, 4L)
new SqlTypedParamValue(DataType.LONG.typeName, 1L),
new SqlTypedParamValue(DataType.LONG.typeName, 2L),
new SqlTypedParamValue(DataType.LONG.typeName, 3L),
new SqlTypedParamValue(DataType.LONG.typeName, 4L)
));
assertThat(expression, instanceOf(Sub.class));
Sub sub = (Sub) expression;
Expand All @@ -62,9 +62,9 @@ public void testMultipleParameters() {
public void testNotEnoughParameters() {
ParsingException ex = expectThrows(ParsingException.class,
() -> new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList(
new SqlTypedParamValue(DataType.LONG.esType, 1L),
new SqlTypedParamValue(DataType.LONG.esType, 2L),
new SqlTypedParamValue(DataType.LONG.esType, 3L)
new SqlTypedParamValue(DataType.LONG.typeName, 1L),
new SqlTypedParamValue(DataType.LONG.typeName, 2L),
new SqlTypedParamValue(DataType.LONG.typeName, 3L)
)));
assertThat(ex.getMessage(), containsString("Not enough actual parameters"));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,9 @@ public void testFunctionWithFunctionWithArg() {
public void testFunctionWithFunctionWithArgAndParams() {
String e = "POWER(?, {fn POWER({fn ABS(?)}, {fN ABS(?)})})";
Function f = (Function) parser.createExpression(e,
asList(new SqlTypedParamValue(DataType.LONG.esType, 1),
new SqlTypedParamValue(DataType.LONG.esType, 1),
new SqlTypedParamValue(DataType.LONG.esType, 1)));
asList(new SqlTypedParamValue(DataType.LONG.typeName, 1),
new SqlTypedParamValue(DataType.LONG.typeName, 1),
new SqlTypedParamValue(DataType.LONG.typeName, 1)));

assertEquals(e, f.sourceText());
assertEquals(2, f.arguments().size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@
import org.elasticsearch.xpack.sql.type.DataType;

import static java.util.Collections.singletonList;
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;

import static org.elasticsearch.common.logging.LoggerMessageFormat.format;

public class LikeEscapingParsingTests extends ESTestCase {

private final SqlParser parser = new SqlParser();
Expand All @@ -33,7 +32,7 @@ private LikePattern like(String pattern) {
Expression exp = null;
boolean parameterized = randomBoolean();
if (parameterized) {
exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.esType, pattern)));
exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.typeName, pattern)));
} else {
exp = parser.createExpression(format(null, "exp LIKE '{}'", pattern));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ public void testSysTablesTypesEnumerationWoString() throws Exception {
}

private SqlTypedParamValue param(Object value) {
return new SqlTypedParamValue(DataTypes.fromJava(value).esType, value);
return new SqlTypedParamValue(DataTypes.fromJava(value).typeName, value);
}

private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> params) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -632,16 +632,16 @@ public void testTopHitsAggregationWithOneArg() {
"\"sort\":[{\"keyword\":{\"order\":\"asc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}"));
}
{
PhysicalPlan p = optimizeAndPlan("SELECT LAST(keyword) FROM test");
PhysicalPlan p = optimizeAndPlan("SELECT LAST(date) FROM test");
assertEquals(EsQueryExec.class, p.getClass());
EsQueryExec eqe = (EsQueryExec) p;
assertEquals(1, eqe.output().size());
assertEquals("LAST(keyword)", eqe.output().get(0).qualifiedName());
assertTrue(eqe.output().get(0).dataType() == DataType.KEYWORD);
assertEquals("LAST(date)", eqe.output().get(0).qualifiedName());
assertTrue(eqe.output().get(0).dataType() == DataType.DATETIME);
assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
endsWith("\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false," +
"\"explain\":false,\"docvalue_fields\":[{\"field\":\"keyword\"}]," +
"\"sort\":[{\"keyword\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}"));
"\"explain\":false,\"docvalue_fields\":[{\"field\":\"date\",\"format\":\"epoch_millis\"}]," +
"\"sort\":[{\"date\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"date\"}}]}}}}}"));
}
}

Expand All @@ -661,17 +661,17 @@ public void testTopHitsAggregationWithTwoArgs() {

}
{
PhysicalPlan p = optimizeAndPlan("SELECT LAST(keyword, int) FROM test");
PhysicalPlan p = optimizeAndPlan("SELECT LAST(date, int) FROM test");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was this change required by the types change in this PR or you just wanted some diversity in FIRST/LAST tests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

assertEquals(EsQueryExec.class, p.getClass());
EsQueryExec eqe = (EsQueryExec) p;
assertEquals(1, eqe.output().size());
assertEquals("LAST(keyword, int)", eqe.output().get(0).qualifiedName());
assertTrue(eqe.output().get(0).dataType() == DataType.KEYWORD);
assertEquals("LAST(date, int)", eqe.output().get(0).qualifiedName());
assertTrue(eqe.output().get(0).dataType() == DataType.DATETIME);
assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
endsWith("\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false," +
"\"explain\":false,\"docvalue_fields\":[{\"field\":\"keyword\"}]," +
"\"explain\":false,\"docvalue_fields\":[{\"field\":\"date\",\"format\":\"epoch_millis\"}]," +
"\"sort\":[{\"int\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"integer\"}}," +
"{\"keyword\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}"));
"{\"date\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"date\"}}]}}}}}"));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ public void testCommonType() {
public void testEsDataTypes() {
for (DataType type : values()) {
if (type != DATE) { // Doesn't have a corresponding type in ES
assertEquals(type, fromTypeName(type.esType));
assertEquals(type, fromTypeName(type.typeName));
}
}
}
Expand Down
Loading