[NSE-955] Support more date format in unix timestamp (#1063)

* Initial commit * Add a UT * Change arrow branch [will revert at last] * Revert "Change arrow branch [will revert at last]" This reverts commit e55df88.
oap-project · Aug 12, 2022 · 875275d · 875275d
1 parent 9567587
commit 875275d
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala b/...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala
@@ -456,6 +456,7 @@ object ColumnarDateTimeExpressions {
     val yearMonthDayFormat = "yyyy-MM-dd"
     val yearMonthDayTimeFormat = "yyyy-MM-dd HH:mm:ss"
     val yearMonthDayTimeNoSepFormat = "yyyyMMddHHmmss"
+    val yearMonthDayNoSepFormat = "yyyyMMdd"
     var formatLiteral: String = null
 
     buildCheck()
@@ -474,7 +475,8 @@ object ColumnarDateTimeExpressions {
             // Only support yyyy-MM-dd or yyyy-MM-dd HH:mm:ss.
             if (!this.formatLiteral.equals(yearMonthDayFormat) &&
               !this.formatLiteral.equals(yearMonthDayTimeFormat) &&
-              !this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) {
+              !this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) &&
+              !this.formatLiteral.equals(yearMonthDayNoSepFormat)) {
               throw new UnsupportedOperationException(
                 s"$formatLiteral is not supported in ColumnarUnixTimestamp.")
             }
@@ -518,7 +520,8 @@ object ColumnarDateTimeExpressions {
           TreeBuilder.makeFunction("divide", Lists.newArrayList(
             ConverterUtils.subtractTimestampOffset(castNode),
             TreeBuilder.makeLiteral(java.lang.Long.valueOf(1000L))), outType)
-        } else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) {
+        } else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) ||
+          this.formatLiteral.equals(yearMonthDayNoSepFormat)) {
           val timestampType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")
           val timestampNode = TreeBuilder.makeFunction("castTIMESTAMP_withCarrying_withoutSep",
             Lists.newArrayList(leftNode), timestampType)

diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala
@@ -785,6 +785,21 @@ class DateTimeSuite extends QueryTest with SharedSparkSession {
         Seq(Row(java.lang.Long.valueOf(1248940800L)),
           Row(java.lang.Long.valueOf(1249027200L)),
           Row(java.lang.Long.valueOf(1249113600L))))
+
+      // Test date format: yyyyMMdd
+      val datesNoSep = Seq("20090730", "20090731", "20090801")
+        .map(s => Tuple1(s)).toDF("time")
+      datesNoSep.createOrReplaceTempView("dates_no_sep")
+      val frameNoSep = sql("SELECT unix_timestamp(time, 'yyyyMMdd') FROM dates_no_sep")
+      frameNoSep.explain()
+      frameNoSep.show()
+      assert(frameNoSep.queryExecution.executedPlan.find(p => p
+        .isInstanceOf[ColumnarConditionProjectExec]).isDefined)
+      checkAnswer(
+        frameNoSep,
+        Seq(Row(java.lang.Long.valueOf(1248940800L)),
+          Row(java.lang.Long.valueOf(1249027200L)),
+          Row(java.lang.Long.valueOf(1249113600L))))
     }
   }
 }