From f5f5aab93229de9de03c103354efccbcd6a95ec0 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 5 Sep 2022 14:42:29 +0800 Subject: [PATCH 1/5] allow to use differnent case in SQL Signed-off-by: Yuan Zhou --- .../oap/expression/ColumnarExpressionConverter.scala | 2 +- .../intel/oap/expression/ColumnarHashAggregation.scala | 10 +++------- .../com/intel/oap/expression/ConverterUtils.scala | 7 +++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index f79afab7f..8187d8f13 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -58,7 +58,7 @@ object ColumnarExpressionConverter extends Logging { val b = bindReference.asInstanceOf[BoundReference] new ColumnarBoundReference(b.ordinal, b.dataType, b.nullable) } else { - return new ColumnarAttributeReference(a.name, a.dataType, a.nullable, a.metadata)( + return new ColumnarAttributeReference(a.name.toLowerCase, a.dataType, a.nullable, a.metadata)( a.exprId, a.qualifier) } diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala index 70868ee3f..b863b7dbc 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala @@ -455,8 +455,7 @@ class ColumnarHashAggregation( } val originalInputFieldList = originalInputAttributes.toList.map(attr => { - Field - .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType)) + ConverterUtils.createArrowField(attr) }) //////////////// Project original input to aggregateExpression input ////////////////// @@ -501,7 +500,7 @@ class ColumnarHashAggregation( val inputAttrs = originalInputAttributes.zipWithIndex .filter { case (attr, i) => - !groupingAttributes.contains(attr) && !partialProjectOrdinalList.toList.contains(i) + !groupingAttributes.contains(attr.withName(attr.name.toLowerCase)) && !partialProjectOrdinalList.toList.contains(i) } .map(_._1) inputAttrQueue = scala.collection.mutable.Queue(inputAttrs: _*) @@ -516,10 +515,7 @@ class ColumnarHashAggregation( val aggregateAttributeFieldList = allAggregateResultAttributes.map(attr => { - Field - .nullable( - s"${attr.name}#${attr.exprId.id}", - CodeGeneration.getResultType(attr.dataType)) + ConverterUtils.createArrowField(attr) }) val nativeFuncNodes = groupingNativeFuncNodes ::: aggrNativeFuncNodes diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala index 65531681a..06dcec518 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala @@ -325,7 +325,7 @@ object ConverterUtils extends Logging { case a: AggregateExpression => getAttrFromExpr(a.aggregateFunction.children(0)) case a: AttributeReference => - a + a.withName(a.name.toLowerCase) case a: Alias => if (skipAlias) { if (a.child.isInstanceOf[AttributeReference] || a.child.isInstanceOf[Coalesce]) { @@ -483,8 +483,7 @@ object ConverterUtils extends Logging { def toArrowSchema(attributes: Seq[Attribute]): Schema = { val fields = attributes.map(attr => { - Field - .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType)) + createArrowField(attr) }) new Schema(fields.toList.asJava) } @@ -618,7 +617,7 @@ object ConverterUtils extends Logging { } def createArrowField(attr: Attribute): Field = - createArrowField(s"${attr.name}#${attr.exprId.id}", attr.dataType) + createArrowField(s"${attr.name.toLowerCase}#${attr.exprId.id}", attr.dataType) private def asTimestampType(inType: ArrowType): ArrowType.Timestamp = { if (inType.getTypeID != ArrowTypeID.Timestamp) { From 32dfc628fdc07775407779fa14f97e135f58aeb2 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Mon, 5 Sep 2022 15:12:12 +0800 Subject: [PATCH 2/5] fix window Signed-off-by: Yuan Zhou --- .../com/intel/oap/execution/ColumnarWindowExec.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala index 93b9a9903..0abfdb7bf 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala @@ -245,8 +245,7 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], val attr = ConverterUtils.getAttrFromExpr(orderSpec.head.child, true) TreeBuilder.makeFunction(row_number_func, List(TreeBuilder.makeField( - Field.nullable(attr.name, - CodeGeneration.getResultType(attr.dataType)))).toList.asJava, + ConverterUtils.createArrowField(attr))).toList.asJava, NoneType.NONE_TYPE ) case (n, f) => @@ -255,8 +254,7 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], .flatMap { case a: AttributeReference => Some(TreeBuilder.makeField( - Field.nullable(a.name, - CodeGeneration.getResultType(a.dataType)))) + ConverterUtils.createArrowField(a))) case c: Cast if c.child.isInstanceOf[AttributeReference] => Some(TreeBuilder.makeField( Field.nullable(c.child.asInstanceOf[AttributeReference].name, @@ -281,8 +279,7 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], val gPartitionSpec = TreeBuilder.makeFunction("partitionSpec", groupingExpressions.map(e => TreeBuilder.makeField( - Field.nullable(e.name, - CodeGeneration.getResultType(e.dataType)))).toList.asJava, + ConverterUtils.createArrowField(e))).toList.asJava, NoneType.NONE_TYPE) // Workaround: // Gandiva doesn't support serializing Struct type so far. Use a fake Binary type instead. From 44d6f30fb71282a4edd85217ad5c364131febd28 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 6 Sep 2022 10:28:55 +0800 Subject: [PATCH 3/5] Revert "fix window" This reverts commit 32dfc628fdc07775407779fa14f97e135f58aeb2. --- .../com/intel/oap/execution/ColumnarWindowExec.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala index 0abfdb7bf..93b9a9903 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala @@ -245,7 +245,8 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], val attr = ConverterUtils.getAttrFromExpr(orderSpec.head.child, true) TreeBuilder.makeFunction(row_number_func, List(TreeBuilder.makeField( - ConverterUtils.createArrowField(attr))).toList.asJava, + Field.nullable(attr.name, + CodeGeneration.getResultType(attr.dataType)))).toList.asJava, NoneType.NONE_TYPE ) case (n, f) => @@ -254,7 +255,8 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], .flatMap { case a: AttributeReference => Some(TreeBuilder.makeField( - ConverterUtils.createArrowField(a))) + Field.nullable(a.name, + CodeGeneration.getResultType(a.dataType)))) case c: Cast if c.child.isInstanceOf[AttributeReference] => Some(TreeBuilder.makeField( Field.nullable(c.child.asInstanceOf[AttributeReference].name, @@ -279,7 +281,8 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], val gPartitionSpec = TreeBuilder.makeFunction("partitionSpec", groupingExpressions.map(e => TreeBuilder.makeField( - ConverterUtils.createArrowField(e))).toList.asJava, + Field.nullable(e.name, + CodeGeneration.getResultType(e.dataType)))).toList.asJava, NoneType.NONE_TYPE) // Workaround: // Gandiva doesn't support serializing Struct type so far. Use a fake Binary type instead. From cbee53e90930fa53c60796902fc992508457d7a7 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 6 Sep 2022 11:44:28 +0800 Subject: [PATCH 4/5] new fix for window Signed-off-by: Yuan Zhou --- .../intel/oap/execution/ColumnarWindowExec.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala index 93b9a9903..e47eeb72a 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala @@ -254,13 +254,15 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], f.children .flatMap { case a: AttributeReference => + val attr = ConverterUtils.getAttrFromExpr(a) Some(TreeBuilder.makeField( - Field.nullable(a.name, - CodeGeneration.getResultType(a.dataType)))) + Field.nullable(attr.name, + CodeGeneration.getResultType(attr.dataType)))) case c: Cast if c.child.isInstanceOf[AttributeReference] => + val attr = ConverterUtils.getAttrFromExpr(c) Some(TreeBuilder.makeField( - Field.nullable(c.child.asInstanceOf[AttributeReference].name, - CodeGeneration.getResultType(c.dataType)))) + Field.nullable(attr.name, + CodeGeneration.getResultType(attr.dataType)))) case _: Cast | _ : Literal => None case _ => @@ -270,9 +272,9 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression], } val groupingExpressions: Seq[AttributeReference] = partitionSpec.map{ case a: AttributeReference => - a + ConverterUtils.getAttrFromExpr(a) case c: Cast if c.child.isInstanceOf[AttributeReference] => - c.child.asInstanceOf[AttributeReference] + ConverterUtils.getAttrFromExpr(c) case _: Cast | _ : Literal => null case _ => From af2af671b180c26f0288f9d2f9938cce982b3307 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Wed, 14 Sep 2022 15:14:09 +0800 Subject: [PATCH 5/5] fix join Signed-off-by: Yuan Zhou --- .../ColumnarConditionedProbeJoin.scala | 17 ++++++----------- .../ColumnarExpressionConverter.scala | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala index 8233c2ffa..a27119208 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala @@ -57,15 +57,14 @@ object ColumnarConditionedProbeJoin extends Logging { buildInputAttributes: Seq[Attribute], builder_type: Int = 1, is_broadcast: Boolean = false): TreeNode = { - val buildInputFieldList: List[Field] = buildInputAttributes.toList.map(attr => { - Field - .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType)) + val buildInputAttrList: List[Attribute] = buildInputAttributes.toList.map(attr => { + attr.withName(attr.name.toLowerCase) }) val buildKeysFunctionList: List[TreeNode] = buildKeys.toList.map(expr => { val (nativeNode, returnType) = if (!is_broadcast) { ConverterUtils.getColumnarFuncNode(expr) } else { - ConverterUtils.getColumnarFuncNode(expr, buildInputAttributes) + ConverterUtils.getColumnarFuncNode(expr, buildInputAttrList) } if (s"${nativeNode.toProtobuf}".contains("none#")) { throw new UnsupportedOperationException( @@ -105,12 +104,10 @@ object ColumnarConditionedProbeJoin extends Logging { builder_type: Int = 0, isNullAwareAntiJoin: Boolean = false): TreeNode = { val buildInputFieldList: List[Field] = buildInputAttributes.toList.map(attr => { - Field - .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType)) + ConverterUtils.createArrowField(attr) }) val streamInputFieldList: List[Field] = streamInputAttributes.toList.map(attr => { - Field - .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType)) + ConverterUtils.createArrowField(attr) }) val buildKeysFunctionList: List[TreeNode] = buildKeys.toList.map(expr => { @@ -132,9 +129,7 @@ object ColumnarConditionedProbeJoin extends Logging { }) val resultFunctionList: List[TreeNode] = output.toList.map(field => { - val field_node = Field.nullable( - s"${field.name}#${field.exprId.id}", - CodeGeneration.getResultType(field.dataType)) + val field_node = ConverterUtils.createArrowField(field) TreeBuilder.makeField(field_node) }) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index 8187d8f13..16dfa91cc 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -48,7 +48,7 @@ object ColumnarExpressionConverter extends Logging { BindReferences.bindReference(expr, attributeSeq, true) if (bindReference == expr) { if (expIdx == -1) { - return new ColumnarAttributeReference(a.name, a.dataType, a.nullable, a.metadata)( + return new ColumnarAttributeReference(a.name.toLowerCase, a.dataType, a.nullable, a.metadata)( a.exprId, a.qualifier) } else {