From 90d796f334292fdb85eb4efc137dd03f17202432 Mon Sep 17 00:00:00 2001 From: William Shen Date: Tue, 10 Sep 2019 22:29:45 -0700 Subject: [PATCH] remove presto internal usage of legacy type syntax (<>) --- .../operator/aggregation/MapUnionAggregation.java | 2 +- .../operator/annotations/ImplementationDependency.java | 2 +- .../io/prestosql/operator/scalar/Re2JRegexpFunctions.java | 6 +++--- .../io/prestosql/operator/scalar/SplitToMapFunction.java | 6 +++--- .../src/main/java/io/prestosql/type/FunctionType.java | 2 +- .../io/prestosql/type/setdigest/SetDigestFunctions.java | 2 +- .../java/io/prestosql/metadata/TestSignatureBinder.java | 2 -- .../src/test/java/io/prestosql/type/TestFunctionType.java | 4 ++-- .../prestosql/plugin/ml/LearnClassifierAggregation.java | 2 +- .../plugin/ml/LearnLibSvmClassifierAggregation.java | 2 +- .../ml/LearnLibSvmVarcharClassifierAggregation.java | 2 +- .../plugin/ml/LearnVarcharClassifierAggregation.java | 2 +- .../src/main/java/io/prestosql/plugin/ml/MLFunctions.java | 8 ++++---- .../etc/catalog/kafka/structural_datatype_avro.json | 4 ++-- .../legacy/systemtables/TableMetadataSystemTable.java | 2 +- .../java/io/prestosql/decoder/avro/TestAvroDecoder.java | 6 +++--- 16 files changed, 26 insertions(+), 28 deletions(-) diff --git a/presto-main/src/main/java/io/prestosql/operator/aggregation/MapUnionAggregation.java b/presto-main/src/main/java/io/prestosql/operator/aggregation/MapUnionAggregation.java index 5277272025a76..4859d003a354c 100644 --- a/presto-main/src/main/java/io/prestosql/operator/aggregation/MapUnionAggregation.java +++ b/presto-main/src/main/java/io/prestosql/operator/aggregation/MapUnionAggregation.java @@ -53,7 +53,7 @@ public class MapUnionAggregation public MapUnionAggregation() { - super(NAME, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), parseTypeSignature("map"), ImmutableList.of(parseTypeSignature("map"))); + super(NAME, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), parseTypeSignature("map(K,V)"), ImmutableList.of(parseTypeSignature("map(K,V)"))); } @Override diff --git a/presto-main/src/main/java/io/prestosql/operator/annotations/ImplementationDependency.java b/presto-main/src/main/java/io/prestosql/operator/annotations/ImplementationDependency.java index 5eae0c61d5dcf..90086214fc06d 100644 --- a/presto-main/src/main/java/io/prestosql/operator/annotations/ImplementationDependency.java +++ b/presto-main/src/main/java/io/prestosql/operator/annotations/ImplementationDependency.java @@ -73,7 +73,7 @@ static void validateImplementationDependencyAnnotation(AnnotatedElement element, static void checkTypeParameters(TypeSignature typeSignature, Set typeParameterNames, AnnotatedElement element) { - // Check recursively if `typeSignature` contains something like `T` + // Check recursively if `typeSignature` contains something like `T(bigint)` if (typeParameterNames.contains(typeSignature.getBase())) { checkArgument(typeSignature.getParameters().isEmpty(), "Expected type parameter not to take parameters, but got %s on method [%s]", typeSignature.getBase(), element); return; diff --git a/presto-main/src/main/java/io/prestosql/operator/scalar/Re2JRegexpFunctions.java b/presto-main/src/main/java/io/prestosql/operator/scalar/Re2JRegexpFunctions.java index 64ec9d1e2d85e..75146184141b3 100644 --- a/presto-main/src/main/java/io/prestosql/operator/scalar/Re2JRegexpFunctions.java +++ b/presto-main/src/main/java/io/prestosql/operator/scalar/Re2JRegexpFunctions.java @@ -66,7 +66,7 @@ public static Slice regexpReplace(@SqlType("varchar(x)") Slice source, @SqlType( @Description("string(s) extracted using the given pattern") @ScalarFunction @LiteralParameters("x") - @SqlType("array") + @SqlType("array(varchar(x))") public static Block regexpExtractAll(@SqlType("varchar(x)") Slice source, @SqlType(Re2JRegexpType.NAME) Re2JRegexp pattern) { return regexpExtractAll(source, pattern, 0); @@ -75,7 +75,7 @@ public static Block regexpExtractAll(@SqlType("varchar(x)") Slice source, @SqlTy @Description("group(s) extracted using the given pattern") @ScalarFunction @LiteralParameters("x") - @SqlType("array") + @SqlType("array(varchar(x))") public static Block regexpExtractAll(@SqlType("varchar(x)") Slice source, @SqlType(Re2JRegexpType.NAME) Re2JRegexp pattern, @SqlType(StandardTypes.BIGINT) long groupIndex) { return pattern.extractAll(source, groupIndex); @@ -104,7 +104,7 @@ public static Slice regexpExtract(@SqlType("varchar(x)") Slice source, @SqlType( @ScalarFunction @Description("returns array of strings split by pattern") @LiteralParameters("x") - @SqlType("array") + @SqlType("array(varchar(x))") public static Block regexpSplit(@SqlType("varchar(x)") Slice source, @SqlType(Re2JRegexpType.NAME) Re2JRegexp pattern) { return pattern.split(source); diff --git a/presto-main/src/main/java/io/prestosql/operator/scalar/SplitToMapFunction.java b/presto-main/src/main/java/io/prestosql/operator/scalar/SplitToMapFunction.java index 26a582c9f0002..b5d502915684b 100644 --- a/presto-main/src/main/java/io/prestosql/operator/scalar/SplitToMapFunction.java +++ b/presto-main/src/main/java/io/prestosql/operator/scalar/SplitToMapFunction.java @@ -41,13 +41,13 @@ public class SplitToMapFunction { private final PageBuilder pageBuilder; - public SplitToMapFunction(@TypeParameter("map") Type mapType) + public SplitToMapFunction(@TypeParameter("map(varchar,varchar)") Type mapType) { pageBuilder = new PageBuilder(ImmutableList.of(mapType)); } - @SqlType("map") - public Block splitToMap(@TypeParameter("map") Type mapType, @SqlType(StandardTypes.VARCHAR) Slice string, @SqlType(StandardTypes.VARCHAR) Slice entryDelimiter, @SqlType(StandardTypes.VARCHAR) Slice keyValueDelimiter) + @SqlType("map(varchar,varchar)") + public Block splitToMap(@TypeParameter("map(varchar,varchar)") Type mapType, @SqlType(StandardTypes.VARCHAR) Slice string, @SqlType(StandardTypes.VARCHAR) Slice entryDelimiter, @SqlType(StandardTypes.VARCHAR) Slice keyValueDelimiter) { checkCondition(entryDelimiter.length() > 0, INVALID_FUNCTION_ARGUMENT, "entryDelimiter is empty"); checkCondition(keyValueDelimiter.length() > 0, INVALID_FUNCTION_ARGUMENT, "keyValueDelimiter is empty"); diff --git a/presto-main/src/main/java/io/prestosql/type/FunctionType.java b/presto-main/src/main/java/io/prestosql/type/FunctionType.java index 76c263b38391f..55623a8226a2e 100644 --- a/presto-main/src/main/java/io/prestosql/type/FunctionType.java +++ b/presto-main/src/main/java/io/prestosql/type/FunctionType.java @@ -86,7 +86,7 @@ public String getDisplayName() ImmutableList names = getTypeParameters().stream() .map(Type::getDisplayName) .collect(toImmutableList()); - return "function<" + Joiner.on(",").join(names) + ">"; + return "function(" + Joiner.on(",").join(names) + ")"; } @Override diff --git a/presto-main/src/main/java/io/prestosql/type/setdigest/SetDigestFunctions.java b/presto-main/src/main/java/io/prestosql/type/setdigest/SetDigestFunctions.java index ae4e6d42459c8..91b3c12a76277 100644 --- a/presto-main/src/main/java/io/prestosql/type/setdigest/SetDigestFunctions.java +++ b/presto-main/src/main/java/io/prestosql/type/setdigest/SetDigestFunctions.java @@ -85,7 +85,7 @@ public static double jaccardIndex(@SqlType(SetDigestType.NAME) Slice slice1, @Sq @ScalarFunction @SqlType("map(bigint,smallint)") - public static Block hashCounts(@TypeParameter("map") Type mapType, @SqlType(SetDigestType.NAME) Slice slice) + public static Block hashCounts(@TypeParameter("map(bigint,smallint)") Type mapType, @SqlType(SetDigestType.NAME) Slice slice) { SetDigest digest = SetDigest.newInstance(slice); diff --git a/presto-main/src/test/java/io/prestosql/metadata/TestSignatureBinder.java b/presto-main/src/test/java/io/prestosql/metadata/TestSignatureBinder.java index 1f8640b2bff28..b0159741d2838 100644 --- a/presto-main/src/test/java/io/prestosql/metadata/TestSignatureBinder.java +++ b/presto-main/src/test/java/io/prestosql/metadata/TestSignatureBinder.java @@ -1068,9 +1068,7 @@ public void testBindParameters() assertThat("T2", boundVariables, "bigint"); assertThat("array(T1)", boundVariables, "array(double)"); assertThat("array(T3)", boundVariables, "array(decimal(5,3))"); - assertThat("array", boundVariables, "array(double)"); assertThat("map(T1,T2)", boundVariables, "map(double,bigint)"); - assertThat("map", boundVariables, "map(double,bigint)"); assertThat("bla(T1,42,T2)", boundVariables, "bla(double,42,bigint)"); assertThat("varchar(p)", boundVariables, "varchar(1)"); assertThat("char(p)", boundVariables, "char(1)"); diff --git a/presto-main/src/test/java/io/prestosql/type/TestFunctionType.java b/presto-main/src/test/java/io/prestosql/type/TestFunctionType.java index faf2eb34c1b5a..62f0e85c28ec4 100644 --- a/presto-main/src/test/java/io/prestosql/type/TestFunctionType.java +++ b/presto-main/src/test/java/io/prestosql/type/TestFunctionType.java @@ -25,7 +25,7 @@ public class TestFunctionType @Test public void testDisplayName() { - Type function = createTestMetadataManager().getType(TypeSignature.parseTypeSignature("function")); - assertEquals(function.getDisplayName(), "function"); + Type function = createTestMetadataManager().getType(TypeSignature.parseTypeSignature("function(row(field double),bigint)")); + assertEquals(function.getDisplayName(), "function(row(field double),bigint)"); } } diff --git a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnClassifierAggregation.java b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnClassifierAggregation.java index 15eec1af1e9ad..a92d748547f18 100644 --- a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnClassifierAggregation.java +++ b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnClassifierAggregation.java @@ -55,7 +55,7 @@ public static void combine(@AggregationState LearnState state, @AggregationState throw new UnsupportedOperationException("LEARN must run on a single machine"); } - @OutputFunction("Classifier") + @OutputFunction("Classifier(bigint)") public static void output(@AggregationState LearnState state, BlockBuilder out) { LearnLibSvmClassifierAggregation.output(state, out); diff --git a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmClassifierAggregation.java b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmClassifierAggregation.java index 5c16e2f54c4b9..3544bdfa325f4 100644 --- a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmClassifierAggregation.java +++ b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmClassifierAggregation.java @@ -65,7 +65,7 @@ public static void combine(@AggregationState LearnState state, @AggregationState throw new UnsupportedOperationException("LEARN must run on a single machine"); } - @OutputFunction("Classifier") + @OutputFunction("Classifier(bigint)") public static void output(@AggregationState LearnState state, BlockBuilder out) { Dataset dataset = new Dataset(state.getLabels(), state.getFeatureVectors(), state.getLabelEnumeration().inverse()); diff --git a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmVarcharClassifierAggregation.java b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmVarcharClassifierAggregation.java index 84fa9aec91b31..d5729a6b62621 100644 --- a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmVarcharClassifierAggregation.java +++ b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnLibSvmVarcharClassifierAggregation.java @@ -51,7 +51,7 @@ public static void combine(@AggregationState LearnState state, @AggregationState throw new UnsupportedOperationException("LEARN must run on a single machine"); } - @OutputFunction("Classifier") + @OutputFunction("Classifier(varchar)") public static void output(@AggregationState LearnState state, BlockBuilder out) { Dataset dataset = new Dataset(state.getLabels(), state.getFeatureVectors(), state.getLabelEnumeration().inverse()); diff --git a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnVarcharClassifierAggregation.java b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnVarcharClassifierAggregation.java index b72f7e62a7e91..2191d129d100a 100644 --- a/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnVarcharClassifierAggregation.java +++ b/presto-ml/src/main/java/io/prestosql/plugin/ml/LearnVarcharClassifierAggregation.java @@ -46,7 +46,7 @@ public static void combine(@AggregationState LearnState state, @AggregationState throw new UnsupportedOperationException("LEARN must run on a single machine"); } - @OutputFunction("Classifier") + @OutputFunction("Classifier(varchar)") public static void output(@AggregationState LearnState state, BlockBuilder out) { LearnLibSvmVarcharClassifierAggregation.output(state, out); diff --git a/presto-ml/src/main/java/io/prestosql/plugin/ml/MLFunctions.java b/presto-ml/src/main/java/io/prestosql/plugin/ml/MLFunctions.java index d125fa806e904..a374c0afb1133 100644 --- a/presto-ml/src/main/java/io/prestosql/plugin/ml/MLFunctions.java +++ b/presto-ml/src/main/java/io/prestosql/plugin/ml/MLFunctions.java @@ -40,22 +40,22 @@ private MLFunctions() @ScalarFunction("classify") @SqlType(StandardTypes.VARCHAR) - public static Slice varcharClassify(@SqlType(MAP_BIGINT_DOUBLE) Block featuresMap, @SqlType("Classifier") Slice modelSlice) + public static Slice varcharClassify(@SqlType(MAP_BIGINT_DOUBLE) Block featuresMap, @SqlType("Classifier(varchar)") Slice modelSlice) { FeatureVector features = ModelUtils.toFeatures(featuresMap); Model model = getOrLoadModel(modelSlice); - checkArgument(model.getType().equals(VARCHAR_CLASSIFIER), "model is not a classifier"); + checkArgument(model.getType().equals(VARCHAR_CLASSIFIER), "model is not a Classifier(varchar)"); Classifier varcharClassifier = (Classifier) model; return Slices.utf8Slice(varcharClassifier.classify(features)); } @ScalarFunction @SqlType(StandardTypes.BIGINT) - public static long classify(@SqlType(MAP_BIGINT_DOUBLE) Block featuresMap, @SqlType("Classifier") Slice modelSlice) + public static long classify(@SqlType(MAP_BIGINT_DOUBLE) Block featuresMap, @SqlType("Classifier(bigint)") Slice modelSlice) { FeatureVector features = ModelUtils.toFeatures(featuresMap); Model model = getOrLoadModel(modelSlice); - checkArgument(model.getType().equals(BIGINT_CLASSIFIER), "model is not a classifier"); + checkArgument(model.getType().equals(BIGINT_CLASSIFIER), "model is not a Classifier(bigint)"); Classifier classifier = (Classifier) model; return classifier.classify(features); } diff --git a/presto-product-tests/conf/presto/etc/catalog/kafka/structural_datatype_avro.json b/presto-product-tests/conf/presto/etc/catalog/kafka/structural_datatype_avro.json index 998d44d32162f..5bbf4233658e0 100644 --- a/presto-product-tests/conf/presto/etc/catalog/kafka/structural_datatype_avro.json +++ b/presto-product-tests/conf/presto/etc/catalog/kafka/structural_datatype_avro.json @@ -8,12 +8,12 @@ "fields": [ { "name": "c_array", - "type": "ARRAY", + "type": "ARRAY(BIGINT)", "mapping": "a_array" }, { "name": "c_map", - "type": "MAP", + "type": "MAP(VARCHAR, VARCHAR)", "mapping": "a_map" } ] diff --git a/presto-raptor-legacy/src/main/java/io/prestosql/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java b/presto-raptor-legacy/src/main/java/io/prestosql/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java index c697a6f03adcc..ce2e51a14817e 100644 --- a/presto-raptor-legacy/src/main/java/io/prestosql/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java +++ b/presto-raptor-legacy/src/main/java/io/prestosql/plugin/raptor/legacy/systemtables/TableMetadataSystemTable.java @@ -78,7 +78,7 @@ public TableMetadataSystemTable(@ForMetadata IDBI dbi, TypeManager typeManager) this.dao = onDemandDao(dbi, MetadataDao.class); requireNonNull(typeManager, "typeManager is null"); - Type arrayOfVarchar = typeManager.getType(parseTypeSignature("array")); + Type arrayOfVarchar = typeManager.getType(parseTypeSignature("array(varchar)")); this.tableMetadata = new ConnectorTableMetadata( new SchemaTableName("system", "tables"), ImmutableList.of( diff --git a/presto-record-decoder/src/test/java/io/prestosql/decoder/avro/TestAvroDecoder.java b/presto-record-decoder/src/test/java/io/prestosql/decoder/avro/TestAvroDecoder.java index 3a6a545324024..ea55426ce9afc 100644 --- a/presto-record-decoder/src/test/java/io/prestosql/decoder/avro/TestAvroDecoder.java +++ b/presto-record-decoder/src/test/java/io/prestosql/decoder/avro/TestAvroDecoder.java @@ -77,9 +77,9 @@ public class TestAvroDecoder private static final AvroRowDecoderFactory DECODER_FACTORY = new AvroRowDecoderFactory(); private static final Metadata METADATA = createTestMetadataManager(); - private static final Type VACHAR_MAP_TYPE = METADATA.getType(parseTypeSignature("map")); - private static final Type DOUBLE_MAP_TYPE = METADATA.getType(parseTypeSignature("map")); - private static final Type REAL_MAP_TYPE = METADATA.getType(parseTypeSignature("map")); + private static final Type VACHAR_MAP_TYPE = METADATA.getType(parseTypeSignature("map(varchar,varchar)")); + private static final Type DOUBLE_MAP_TYPE = METADATA.getType(parseTypeSignature("map(varchar,double)")); + private static final Type REAL_MAP_TYPE = METADATA.getType(parseTypeSignature("map(varchar,real)")); private static String getAvroSchema(String name, String dataType) {