[NSE-400] Fix the bug for negative decimal data (oap-project#686)

* Fix the bug for positive decimal data * Fix format * Fix the unit test for 'SPARK-22348: table cache should do partition batch pruning (whole-stage-codegen on)'
zhouyuan · Jan 10, 2022 · 5b09e24 · 5b09e24
1 parent 0ba160b
commit 5b09e24
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/...e/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/...e/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
@@ -20,8 +20,8 @@ package org.apache.spark.sql.execution.columnar
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 
-import com.intel.oap.execution.ColumnarConditionProjectExec
-import com.intel.oap.sql.execution.RowToArrowColumnarExec
+import com.intel.oap.execution.{ArrowRowToColumnarExec, ColumnarConditionProjectExec}
+
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.InternalRow
@@ -506,7 +506,7 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
     val df2 = df1.where("y = 3")
 
     val planBeforeFilter = df2.queryExecution.executedPlan.collect {
-      case ColumnarConditionProjectExec(_, _, c: RowToArrowColumnarExec) => c.child
+      case ColumnarConditionProjectExec(_, _, c: ArrowRowToColumnarExec) => c.child
       case FilterExec(_, c: ColumnarToRowExec) => c.child
       case WholeStageCodegenExec(FilterExec(_, ColumnarToRowExec(i: InputAdapter))) => i.child
     }

diff --git a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc
@@ -393,6 +393,11 @@ arrow::Status CreateArrayData(std::shared_ptr<arrow::Schema> schema, int64_t num
             for (int k = length - 1; k >= 0; k--) {
               bytesValue2[length - 1 - k] = bytesValue[k];
             }
+            if (int8_t(bytesValue[0]) < 0) {
+              for (int k = length; k < 16; k++) {
+                bytesValue2[k] = 255;
+              }
+            }
             arrow::Decimal128 value =
                 arrow::Decimal128(arrow::BasicDecimal128(bytesValue2));
             array_data[position] = value;
@@ -950,6 +955,11 @@ arrow::Status CreateArrayData(std::shared_ptr<arrow::Schema> schema, int64_t num
                     for (int k = elementLength - 1; k >= 0; k--) {
                       bytesValue2[elementLength - 1 - k] = bytesValue[k];
                     }
+                    if (int8_t(bytesValue[0]) < 0) {
+                      for (int k = elementLength; k < 16; k++) {
+                        bytesValue2[k] = 255;
+                      }
+                    }
                     arrow::Decimal128 value =
                         arrow::Decimal128(arrow::BasicDecimal128(bytesValue2));
                     RETURN_NOT_OK(child_builder.Append(value));