Use JSONUtils.castStringsToFloats

Signed-off-by: Nghia Truong <[email protected]>
NVIDIA · Oct 24, 2024 · e9d6a8c · e9d6a8c
1 parent 94aaf95
commit e9d6a8c
Showing 1 changed file with 3 additions and 23 deletions.
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuJsonReadCommon.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuJsonReadCommon.scala
@@ -24,7 +24,7 @@ import com.fasterxml.jackson.core.JsonParser
 import com.nvidia.spark.rapids.{ColumnCastUtil, GpuColumnVector, GpuScalar, GpuTextBasedPartitionReader}
 import com.nvidia.spark.rapids.Arm.withResource
 import com.nvidia.spark.rapids.RapidsPluginImplicits.AutoCloseableProducingArray
-import com.nvidia.spark.rapids.jni.{CastStrings, JSONUtils}
+import com.nvidia.spark.rapids.jni.{JSONUtils}
 
 import org.apache.spark.sql.catalyst.json.{GpuJsonUtils, JSONOptions}
 import org.apache.spark.sql.rapids.shims.GpuJsonToStructsShim
@@ -63,27 +63,6 @@ object GpuJsonReadCommon {
   }
 
 
-  private def castStringToFloat(cv: ColumnView, dt: DType,
-      options: JSONOptions): ColumnVector = {
-//    TableDebug.get().debug("input", cv)
-
-    if (options.allowNonNumericNumbers) {
-      // The input already parsed non-numeric numbers according to the reading options.
-      // Only quoted strings left.
-      withResource(JSONUtils.removeQuotesForFloats(cv)) { sanitizedInput =>
-//        TableDebug.get().debug("sanitized float", sanitizedInput)
-
-        val out = CastStrings.toFloat(sanitizedInput, false, dt)
-//        TableDebug.get().debug("out with allow", out)
-        out
-      }
-    } else {
-      val out = CastStrings.toFloat(cv, false, dt)
-//      TableDebug.get().debug("out no allow", out)
-      out
-    }
-  }
-
 
 
   private def dateFormat(options: JSONOptions): Option[String] =
@@ -175,7 +154,8 @@ object GpuJsonReadCommon {
       //
       // DONE
       case (cv, Some(dt)) if (dt == DoubleType || dt == FloatType) && cv.getType == DType.STRING =>
-        castStringToFloat(cv,  GpuColumnVector.getNonNestedRapidsType(dt), options)
+        JSONUtils.castStringsToFloats(cv,  GpuColumnVector.getNonNestedRapidsType(dt),
+          options.allowNonNumericNumbers)
       //
       //