[NSE-1108] allow to use different cases in column names (#1098)

* allow to use differnent case in SQL Signed-off-by: Yuan Zhou <[email protected]> * fix window Signed-off-by: Yuan Zhou <[email protected]> * Revert "fix window" This reverts commit 32dfc62. * new fix for window Signed-off-by: Yuan Zhou <[email protected]> * fix join note: this fix relies on lower case table schema, a better solution would be switching to the boundreference: https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala#L69
oap-project · Sep 14, 2022 · f834def · f834def
1 parent e58a4b3
commit f834def
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 30 deletions.
diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/execution/ColumnarWindowExec.scala
@@ -254,13 +254,15 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression],
             f.children
               .flatMap {
                 case a: AttributeReference =>
+                  val attr = ConverterUtils.getAttrFromExpr(a)
                   Some(TreeBuilder.makeField(
-                    Field.nullable(a.name,
-                      CodeGeneration.getResultType(a.dataType))))
+                    Field.nullable(attr.name,
+                      CodeGeneration.getResultType(attr.dataType))))
                 case c: Cast if c.child.isInstanceOf[AttributeReference] =>
+                  val attr = ConverterUtils.getAttrFromExpr(c)
                   Some(TreeBuilder.makeField(
-                    Field.nullable(c.child.asInstanceOf[AttributeReference].name,
-                      CodeGeneration.getResultType(c.dataType))))
+                    Field.nullable(attr.name,
+                      CodeGeneration.getResultType(attr.dataType))))
                 case _: Cast | _ : Literal =>
                   None
                 case _ =>
@@ -271,9 +273,9 @@ case class ColumnarWindowExec(windowExpression: Seq[NamedExpression],
         // TODO(yuan): using ConverterUtils.getAttrFromExpr 
         val groupingExpressions: Seq[AttributeReference] = partitionSpec.map{
           case a: AttributeReference =>
-            a
+            ConverterUtils.getAttrFromExpr(a)
           case c: Cast if c.child.isInstanceOf[AttributeReference] =>
-            c.child.asInstanceOf[AttributeReference]
+            ConverterUtils.getAttrFromExpr(c)
           case _: Cast | _ : Literal =>
             null
           case n: KnownFloatingPointNormalized =>

diff --git a/...ql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala b/...ql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarConditionedProbeJoin.scala
@@ -57,15 +57,14 @@ object ColumnarConditionedProbeJoin extends Logging {
       buildInputAttributes: Seq[Attribute],
       builder_type: Int = 1,
       is_broadcast: Boolean = false): TreeNode = {
-    val buildInputFieldList: List[Field] = buildInputAttributes.toList.map(attr => {
-      Field
-        .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType))
+    val buildInputAttrList: List[Attribute] = buildInputAttributes.toList.map(attr => {
+      attr.withName(attr.name.toLowerCase)
     })
     val buildKeysFunctionList: List[TreeNode] = buildKeys.toList.map(expr => {
       val (nativeNode, returnType) = if (!is_broadcast) {
         ConverterUtils.getColumnarFuncNode(expr)
       } else {
-        ConverterUtils.getColumnarFuncNode(expr, buildInputAttributes)
+        ConverterUtils.getColumnarFuncNode(expr, buildInputAttrList)
       }
       if (s"${nativeNode.toProtobuf}".contains("none#")) {
         throw new UnsupportedOperationException(
@@ -105,12 +104,10 @@ object ColumnarConditionedProbeJoin extends Logging {
       builder_type: Int = 0,
       isNullAwareAntiJoin: Boolean = false): TreeNode = {
     val buildInputFieldList: List[Field] = buildInputAttributes.toList.map(attr => {
-      Field
-        .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType))
+      ConverterUtils.createArrowField(attr)
     })
     val streamInputFieldList: List[Field] = streamInputAttributes.toList.map(attr => {
-      Field
-        .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType))
+      ConverterUtils.createArrowField(attr)
     })
 
     val buildKeysFunctionList: List[TreeNode] = buildKeys.toList.map(expr => {
@@ -132,9 +129,7 @@ object ColumnarConditionedProbeJoin extends Logging {
     })
 
     val resultFunctionList: List[TreeNode] = output.toList.map(field => {
-      val field_node = Field.nullable(
-        s"${field.name}#${field.exprId.id}",
-        CodeGeneration.getResultType(field.dataType))
+      val field_node = ConverterUtils.createArrowField(field)
       TreeBuilder.makeField(field_node)
     })
 

diff --git a/...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala
@@ -48,7 +48,7 @@ object ColumnarExpressionConverter extends Logging {
             BindReferences.bindReference(expr, attributeSeq, true)
           if (bindReference == expr) {
             if (expIdx == -1) {
-              return new ColumnarAttributeReference(a.name, a.dataType, a.nullable, a.metadata)(
+              return new ColumnarAttributeReference(a.name.toLowerCase, a.dataType, a.nullable, a.metadata)(
                 a.exprId,
                 a.qualifier)
             } else {
@@ -58,7 +58,7 @@ object ColumnarExpressionConverter extends Logging {
           val b = bindReference.asInstanceOf[BoundReference]
           new ColumnarBoundReference(b.ordinal, b.dataType, b.nullable)
         } else {
-          return new ColumnarAttributeReference(a.name, a.dataType, a.nullable, a.metadata)(
+          return new ColumnarAttributeReference(a.name.toLowerCase, a.dataType, a.nullable, a.metadata)(
             a.exprId,
             a.qualifier)
         }

diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarHashAggregation.scala
@@ -455,8 +455,7 @@ class ColumnarHashAggregation(
     }
 
     val originalInputFieldList = originalInputAttributes.toList.map(attr => {
-      Field
-        .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType))
+      ConverterUtils.createArrowField(attr)
     })
 
     //////////////// Project original input to aggregateExpression input //////////////////
@@ -501,7 +500,7 @@ class ColumnarHashAggregation(
     val inputAttrs = originalInputAttributes.zipWithIndex
       .filter {
         case (attr, i) =>
-          !groupingAttributes.contains(attr) && !partialProjectOrdinalList.toList.contains(i)
+          !groupingAttributes.contains(attr.withName(attr.name.toLowerCase)) && !partialProjectOrdinalList.toList.contains(i)
       }
       .map(_._1)
     inputAttrQueue = scala.collection.mutable.Queue(inputAttrs: _*)
@@ -516,10 +515,7 @@ class ColumnarHashAggregation(
 
     val aggregateAttributeFieldList =
       allAggregateResultAttributes.map(attr => {
-        Field
-          .nullable(
-            s"${attr.name}#${attr.exprId.id}",
-            CodeGeneration.getResultType(attr.dataType))
+        ConverterUtils.createArrowField(attr)
       })
 
     val nativeFuncNodes = groupingNativeFuncNodes ::: aggrNativeFuncNodes

diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ConverterUtils.scala
@@ -325,7 +325,7 @@ object ConverterUtils extends Logging {
       case a: AggregateExpression =>
         getAttrFromExpr(a.aggregateFunction.children(0))
       case a: AttributeReference =>
-        a
+        a.withName(a.name.toLowerCase)
       case a: Alias =>
         if (skipAlias) {
           if (a.child.isInstanceOf[AttributeReference] || a.child.isInstanceOf[Coalesce]) {
@@ -483,8 +483,7 @@ object ConverterUtils extends Logging {
 
   def toArrowSchema(attributes: Seq[Attribute]): Schema = {
     val fields = attributes.map(attr => {
-      Field
-        .nullable(s"${attr.name}#${attr.exprId.id}", CodeGeneration.getResultType(attr.dataType))
+      createArrowField(attr)
     })
     new Schema(fields.toList.asJava)
   }
@@ -630,7 +629,7 @@ object ConverterUtils extends Logging {
   }
 
   def createArrowField(attr: Attribute): Field =
-    createArrowField(s"${attr.name}#${attr.exprId.id}", attr.dataType)
+    createArrowField(s"${attr.name.toLowerCase}#${attr.exprId.id}", attr.dataType)
 
   private def asTimestampType(inType: ArrowType): ArrowType.Timestamp = {
     if (inType.getTypeID != ArrowTypeID.Timestamp) {