fallback timestamp scan for parquet if necessary

apache · Sep 2, 2024 · bf8c310 · bf8c310
1 parent cc86a9a
commit bf8c310
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 25 deletions.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -99,7 +99,15 @@ object VeloxBackendSettings extends BackendSettingsApi {
     }
 
     format match {
-      case ParquetReadFormat | DwrfReadFormat => ValidationResult.succeeded
+      case ParquetReadFormat =>
+        val typeValidator: PartialFunction[StructField, String] = {
+          // Parquet timestamp is not fully supported yet
+          case StructField(_, TimestampType, _, _)
+              if GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled =>
+            "TimestampType"
+        }
+        validateTypes(typeValidator)
+      case DwrfReadFormat => ValidationResult.succeeded
       case OrcReadFormat =>
         if (!GlutenConfig.getConf.veloxOrcScanEnabled) {
           ValidationResult.failed(s"Velox ORC scan is turned off.")

diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -353,10 +353,7 @@ object ExpressionConverter extends SQLConfHelper with Logging {
           // Different backends may have different result.
           BackendsApiManager.getSparkPlanExecApiInstance.genGetStructFieldTransformer(
             substraitExprName,
-            replaceWithExpressionTransformerInternal(
-              getStructField.child,
-              attributeSeq,
-              expressionsMap),
+            replaceWithExpressionTransformer0(getStructField.child, attributeSeq, expressionsMap),
             bindRef.ordinal,
             getStructField)
         } catch {

diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -899,10 +899,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetV1PartitionDiscoverySuite]
   enableSuite[GlutenParquetV2PartitionDiscoverySuite]
   enableSuite[GlutenParquetProtobufCompatibilitySuite]
-    // Velox issue
-    .exclude("unannotated array of struct with unannotated array")
-    .exclude("struct with unannotated array")
-    .exclude("unannotated array of struct")
   enableSuite[GlutenParquetV1QuerySuite]
     // Unsupport spark.sql.files.ignoreCorruptFiles.
     .exclude("Enabling/disabling ignoreCorruptFiles")
@@ -946,7 +942,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetThriftCompatibilitySuite]
     // Rewrite for file locating.
     .exclude("Read Parquet file generated by parquet-thrift")
-    .exclude("SPARK-10136 list of primitive list")
   enableSuite[GlutenFileTableSuite]
   enableSuite[GlutenBucketingUtilsSuite]
   enableSuite[GlutenDataSourceStrategySuite]

diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -717,10 +717,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetV1PartitionDiscoverySuite]
   enableSuite[GlutenParquetV2PartitionDiscoverySuite]
   enableSuite[GlutenParquetProtobufCompatibilitySuite]
-    // Velox issue
-    .exclude("unannotated array of struct with unannotated array")
-    .exclude("struct with unannotated array")
-    .exclude("unannotated array of struct")
   enableSuite[GlutenParquetV1QuerySuite]
     // Unsupport spark.sql.files.ignoreCorruptFiles.
     .exclude("Enabling/disabling ignoreCorruptFiles")
@@ -766,7 +762,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetThriftCompatibilitySuite]
     // Rewrite for file locating.
     .exclude("Read Parquet file generated by parquet-thrift")
-    .exclude("SPARK-10136 list of primitive list")
   enableSuite[GlutenParquetVectorizedSuite]
   enableSuite[GlutenTextV1Suite]
   enableSuite[GlutenTextV2Suite]

diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -701,10 +701,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetV1PartitionDiscoverySuite]
   enableSuite[GlutenParquetV2PartitionDiscoverySuite]
   enableSuite[GlutenParquetProtobufCompatibilitySuite]
-    // Velox issue
-    .exclude("unannotated array of struct with unannotated array")
-    .exclude("struct with unannotated array")
-    .exclude("unannotated array of struct")
   enableSuite[GlutenParquetV1QuerySuite]
     // Unsupport spark.sql.files.ignoreCorruptFiles.
     .exclude("Enabling/disabling ignoreCorruptFiles")
@@ -760,7 +756,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetThriftCompatibilitySuite]
     // Rewrite for file locating.
     .exclude("Read Parquet file generated by parquet-thrift")
-    .exclude("SPARK-10136 list of primitive list")
   enableSuite[GlutenParquetVectorizedSuite]
   enableSuite[GlutenTextV1Suite]
   enableSuite[GlutenTextV2Suite]

diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -706,10 +706,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetV1PartitionDiscoverySuite]
   enableSuite[GlutenParquetV2PartitionDiscoverySuite]
   enableSuite[GlutenParquetProtobufCompatibilitySuite]
-    // Velox issue
-    .exclude("unannotated array of struct with unannotated array")
-    .exclude("struct with unannotated array")
-    .exclude("unannotated array of struct")
   enableSuite[GlutenParquetV1QuerySuite]
     .exclude("row group skipping doesn't overflow when reading into larger type")
     // Unsupport spark.sql.files.ignoreCorruptFiles.
@@ -767,7 +763,6 @@ class VeloxTestSettings extends BackendTestSettings {
   enableSuite[GlutenParquetThriftCompatibilitySuite]
     // Rewrite for file locating.
     .exclude("Read Parquet file generated by parquet-thrift")
-    .exclude("SPARK-10136 list of primitive list")
   enableSuite[GlutenParquetVectorizedSuite]
   enableSuite[GlutenTextV1Suite]
   enableSuite[GlutenTextV2Suite]