Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-955] Support more date format in unix timestamp #1063

Merged
merged 4 commits into from
Aug 12, 2022
Merged

[NSE-955] Support more date format in unix timestamp #1063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f447e28
Initial commit
PHILO-HE Aug 11, 2022
e526d1b
Add a UT
PHILO-HE Aug 11, 2022
e55df88
Change arrow branch [will revert at last]
PHILO-HE Aug 12, 2022
3e9f0ee
Revert "Change arrow branch [will revert at last]"
PHILO-HE Aug 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions ...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,7 @@ object ColumnarDateTimeExpressions {
val yearMonthDayFormat = "yyyy-MM-dd"
val yearMonthDayTimeFormat = "yyyy-MM-dd HH:mm:ss"
val yearMonthDayTimeNoSepFormat = "yyyyMMddHHmmss"
val yearMonthDayNoSepFormat = "yyyyMMdd"
var formatLiteral: String = null

buildCheck()
Expand All @@ -474,7 +475,8 @@ object ColumnarDateTimeExpressions {
// Only support yyyy-MM-dd or yyyy-MM-dd HH:mm:ss.
if (!this.formatLiteral.equals(yearMonthDayFormat) &&
!this.formatLiteral.equals(yearMonthDayTimeFormat) &&
!this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) {
!this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) &&
!this.formatLiteral.equals(yearMonthDayNoSepFormat)) {
throw new UnsupportedOperationException(
s"$formatLiteral is not supported in ColumnarUnixTimestamp.")
}
Expand Down Expand Up @@ -513,7 +515,8 @@ object ColumnarDateTimeExpressions {
TreeBuilder.makeFunction("divide", Lists.newArrayList(
ConverterUtils.subtractTimestampOffset(castNode),
TreeBuilder.makeLiteral(java.lang.Long.valueOf(1000L))), outType)
} else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat)) {
} else if (this.formatLiteral.equals(yearMonthDayTimeNoSepFormat) ||
this.formatLiteral.equals(yearMonthDayNoSepFormat)) {
val timestampType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")
val timestampNode = TreeBuilder.makeFunction("castTIMESTAMP_withCarrying_withoutSep",
Lists.newArrayList(leftNode), timestampType)
Expand Down
15 changes: 15 additions & 0 deletions native-sql-engine/core/src/test/scala/com/intel/oap/misc/DateTimeSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,21 @@ class DateTimeSuite extends QueryTest with SharedSparkSession {
Seq(Row(java.lang.Long.valueOf(1248940800L)),
Row(java.lang.Long.valueOf(1249027200L)),
Row(java.lang.Long.valueOf(1249113600L))))

// Test date format: yyyyMMdd
val datesNoSep = Seq("20090730", "20090731", "20090801")
.map(s => Tuple1(s)).toDF("time")
datesNoSep.createOrReplaceTempView("dates_no_sep")
val frameNoSep = sql("SELECT unix_timestamp(time, 'yyyyMMdd') FROM dates_no_sep")
frameNoSep.explain()
frameNoSep.show()
assert(frameNoSep.queryExecution.executedPlan.find(p => p
.isInstanceOf[ColumnarConditionProjectExec]).isDefined)
checkAnswer(
frameNoSep,
Seq(Row(java.lang.Long.valueOf(1248940800L)),
Row(java.lang.Long.valueOf(1249027200L)),
Row(java.lang.Long.valueOf(1249113600L))))
}
}
}