Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SQL: Scripting support for casting functions CAST and CONVERT #36640

Merged
merged 4 commits into from
Dec 17, 2018
Merged

SQL: Scripting support for casting functions CAST and CONVERT #36640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6ade386
Scripting support for casting functions CAST and CONVERT
astefan Dec 14, 2018
65ce47d
Addressing comments
astefan Dec 14, 2018
84c4839
Merge branch 'master' of https://github.com/elastic/elasticsearch int…
astefan Dec 14, 2018
d271244
Overlooked test
astefan Dec 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 60 additions & 6 deletions x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,66 @@ SELECT MAX(languages) max, MIN(languages) min, SUM(languages) sum, AVG(languages
FROM test_emp GROUP BY languages ORDER BY languages ASC LIMIT 5;

max:bt | min:bt | sum:bt | avg:d | percent:d | percent_rank:d| kurtosis:d | skewness:d
---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------
null |null |null |null |null |null |null |null
1 |1 |15 |1 |1.0 |100.0 |NaN |NaN
2 |2 |38 |2 |2.0 |100.0 |NaN |NaN
3 |3 |51 |3 |3.0 |100.0 |NaN |NaN
4 |4 |72 |4 |4.0 |0.0 |NaN |NaN
---------------+---------------+---------------+--------------+---------------+---------------+---------------+---------------
null |null |null |null |null |null |null |null
1 |1 |15 |1 |1.0 |100.0 |NaN |NaN
2 |2 |38 |2 |2.0 |100.0 |NaN |NaN
3 |3 |51 |3 |3.0 |100.0 |NaN |NaN
4 |4 |72 |4 |4.0 |0.0 |NaN |NaN
;

aggByComplexCastedValue
SELECT CONVERT(CONCAT(LTRIM(CONVERT("emp_no", SQL_VARCHAR)), LTRIM(CONVERT("languages", SQL_VARCHAR))), SQL_BIGINT) AS "TEMP"
FROM "test_emp" GROUP BY "TEMP" ORDER BY "TEMP" LIMIT 20;

TEMP:l
---------------
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
100012
100025
100034
100045
100051
100063
100074
100082
100091
100104
;

aggAndOrderByCastedValue
SELECT CHAR_LENGTH(SPACE(CAST(languages AS SMALLINT))), COUNT(*) FROM test_emp GROUP BY 1 ORDER BY 1 DESC;

CHAR_LENGTH(SPACE(CAST(languages AS SMALLINT))):i| COUNT(1):l
-------------------------------------------------+---------------
5 |21
4 |18
3 |17
2 |19
1 |15
null |10
;

aggAndOrderByCastedFunctionValue
SELECT ROUND(SQRT(CAST(EXP(languages) AS SMALLINT)), 2), COUNT(*) FROM test_emp GROUP BY 1 ORDER BY 1 DESC;

ROUND(SQRT(CAST(EXP(languages) AS SMALLINT)),2):d| COUNT(1):l
-------------------------------------------------+---------------
12.17 |21
7.42 |18
4.47 |17
2.65 |19
1.73 |15
null |10
;


Expand Down
16 changes: 16 additions & 0 deletions ...plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Cast.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@

import org.elasticsearch.xpack.sql.expression.Expression;
import org.elasticsearch.xpack.sql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.sql.tree.Location;
import org.elasticsearch.xpack.sql.tree.NodeInfo;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.DataTypeConversion;
import org.elasticsearch.xpack.sql.type.DataTypes;

import java.util.Locale;
import java.util.Objects;

import static org.elasticsearch.xpack.sql.expression.gen.script.ParamsBuilder.paramsBuilder;

public class Cast extends UnaryScalarFunction {

private final DataType dataType;
Expand Down Expand Up @@ -74,6 +78,18 @@ protected Processor makeProcessor() {
return new CastProcessor(DataTypeConversion.conversionFor(from(), to()));
}

@Override
public ScriptTemplate asScript() {
ScriptTemplate fieldAsScript = asScript(field());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the solution is to use a slightly different script, simply override processScript (see its implementations for examples like MathFunction). This way you only need to define the string and not worry about the boiler-plate around passing the script further.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With processScript we cannot set the dataType as parameter and we are forced to send it as String inline. Will leave it as is at the moment, as discussed.

return new ScriptTemplate(
formatTemplate(String.format(Locale.ROOT, "{sql}.cast(%s,{})", fieldAsScript.template())),
paramsBuilder()
.script(fieldAsScript.params())
.variable(dataType.name())
.build(),
dataType());
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), dataType);
Expand Down
8 changes: 8 additions & 0 deletions .../elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.elasticsearch.xpack.sql.expression.predicate.operator.comparison.InProcessor;
import org.elasticsearch.xpack.sql.expression.predicate.regex.RegexProcessor.RegexOperation;
import org.elasticsearch.xpack.sql.type.DataType;
import org.elasticsearch.xpack.sql.type.DataTypeConversion;
import org.elasticsearch.xpack.sql.util.DateUtils;
import org.elasticsearch.xpack.sql.util.StringUtils;

Expand Down Expand Up @@ -458,4 +459,11 @@ public static String substring(String s, Number start, Number length) {
public static String ucase(String s) {
return (String) StringOperation.UCASE.apply(s);
}

//
// Casting
//
public static Object cast(Object value, String typeName) {
return DataTypeConversion.convert(value, DataType.fromTypeName(typeName));
}
}
5 changes: 5 additions & 0 deletions x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,4 +130,9 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalS
String space(Number)
String substring(String, Number, Number)
String ucase(String)

#
# Casting
#
def cast(Object, String)
}
8 changes: 4 additions & 4 deletions x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,10 +240,10 @@ public void testGroupKeyTypes_IP() {
assertEquals(EsQueryExec.class, p.getClass());
EsQueryExec ee = (EsQueryExec) p;
assertThat(ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
endsWith("{\"script\":{" +
"\"source\":\"InternalSqlScriptUtils.docValue(doc,params.v0)\",\"lang\":\"painless\"," +
"\"params\":{\"v0\":\"keyword\"}},\"missing_bucket\":true," +
"\"value_type\":\"ip\",\"order\":\"asc\"}}}]}}}"));
endsWith("{\"script\":{\"source\":\"InternalSqlScriptUtils.cast(" +
"InternalSqlScriptUtils.docValue(doc,params.v0),params.v1)\"," +
"\"lang\":\"painless\",\"params\":{\"v0\":\"keyword\",\"v1\":\"IP\"}}," +
"\"missing_bucket\":true,\"value_type\":\"ip\",\"order\":\"asc\"}}}]}}}"));
assertEquals(2, ee.output().size());
assertThat(ee.output().get(0).toString(), startsWith("COUNT(1){a->"));
assertThat(ee.output().get(1).toString(), startsWith("a{s->"));
Expand Down