fallback timestamp scan for parquet if necessary

apache · Aug 14, 2024 · ac0a3ac · ac0a3ac
1 parent 9a7aa70
commit ac0a3ac
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -100,7 +100,15 @@ object VeloxBackendSettings extends BackendSettingsApi {
     }
 
     format match {
-      case ParquetReadFormat | DwrfReadFormat => ValidationResult.succeeded
+      case ParquetReadFormat =>
+        val typeValidator: PartialFunction[StructField, String] = {
+          // Parquet timestamp is not fully supported yet
+          case StructField(_, TimestampType, _, _)
+            if GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled =>
+            "TimestampType"
+        }
+        validateTypes(typeValidator)
+      case DwrfReadFormat => ValidationResult.succeeded
       case OrcReadFormat =>
         if (!GlutenConfig.getConf.veloxOrcScanEnabled) {
           ValidationResult.failed(s"Velox ORC scan is turned off.")