From 875275d4c29780d75a5ce4b9a4c88149208defdb Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 12 Aug 2022 14:43:00 +0800 Subject: [PATCH] [NSE-955] Support more date format in unix timestamp (#1063) * Initial commit * Add a UT * Change arrow branch [will revert at last] * Revert "Change arrow branch [will revert at last]" This reverts commit e55df88acf0e96e6a3458b7278d5c326a2044632. --- .../expression/ColumnarDateTimeExpressions.scala | 7 +++++-- .../scala/com/intel/oap/misc/DateTimeSuite.scala | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala index 0f1a82aad..dd0b77a50 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala @@ -456,6 +456,7 @@ object ColumnarDateTimeExpressions { val yearMonthDayFormat = "yyyy-MM-dd" val yearMonthDayTimeFormat = "yyyy-MM-dd HH:mm:ss" val yearMonthDayTimeNoSepFormat = "yyyyMMddHHmmss" + val yearMonthDayNoSepFormat = "yyyyMMdd" var formatLiteral: String = null buildCheck() @@ -474,7 +475,8 @@ object ColumnarDateTimeExpressions { // Only support yyyy-MM-dd or yyyy-MM-dd HH:mm:ss. if (!this.formatLiteral.equals(yearMonthDayFormat) && !this.formatLiteral.equals(yearMonthDayTimeFormat) && - !this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) { + !this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) && + !this.formatLiteral.equals(yearMonthDayNoSepFormat)) { throw new UnsupportedOperationException( s"$formatLiteral is not supported in ColumnarUnixTimestamp.") } @@ -518,7 +520,8 @@ object ColumnarDateTimeExpressions { TreeBuilder.makeFunction("divide", Lists.newArrayList( ConverterUtils.subtractTimestampOffset(castNode), TreeBuilder.makeLiteral(java.lang.Long.valueOf(1000L))), outType) - } else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) { + } else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) || + this.formatLiteral.equals(yearMonthDayNoSepFormat)) { val timestampType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC") val timestampNode = TreeBuilder.makeFunction("castTIMESTAMP_withCarrying_withoutSep", Lists.newArrayList(leftNode), timestampType) diff --git a/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala b/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala index 9778fef9c..3a304e315 100644 --- a/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala +++ b/native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala @@ -785,6 +785,21 @@ class DateTimeSuite extends QueryTest with SharedSparkSession { Seq(Row(java.lang.Long.valueOf(1248940800L)), Row(java.lang.Long.valueOf(1249027200L)), Row(java.lang.Long.valueOf(1249113600L)))) + + // Test date format: yyyyMMdd + val datesNoSep = Seq("20090730", "20090731", "20090801") + .map(s => Tuple1(s)).toDF("time") + datesNoSep.createOrReplaceTempView("dates_no_sep") + val frameNoSep = sql("SELECT unix_timestamp(time, 'yyyyMMdd') FROM dates_no_sep") + frameNoSep.explain() + frameNoSep.show() + assert(frameNoSep.queryExecution.executedPlan.find(p => p + .isInstanceOf[ColumnarConditionProjectExec]).isDefined) + checkAnswer( + frameNoSep, + Seq(Row(java.lang.Long.valueOf(1248940800L)), + Row(java.lang.Long.valueOf(1249027200L)), + Row(java.lang.Long.valueOf(1249113600L)))) } } }