From 759bca62903b8624dda91ef081e93cd4a30969fe Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Wed, 7 Nov 2018 22:40:43 +0300 Subject: [PATCH] Addressing Dongjoon's review comments --- .../sql/catalyst/expressions/CsvExpressionsSuite.scala | 2 +- .../sql/catalyst/expressions/JsonExpressionsSuite.scala | 2 +- .../scala/org/apache/spark/sql/CsvFunctionsSuite.scala | 8 +++----- .../scala/org/apache/spark/sql/JsonFunctionsSuite.scala | 8 +++----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala index 897a3574e04af..f5aaaec456153 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala @@ -211,7 +211,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P ) } - test("take into account locale while parsing date") { + test("parse date with locale") { Seq("en-US", "ru-RU").foreach { langTag => val locale = Locale.forLanguageTag(langTag) val date = new SimpleDateFormat("yyyy-MM-dd").parse("2018-11-05") diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 7c2897c2174cb..6ee8c74010d3d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -739,7 +739,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with "struct") } - test("take into account locale while parsing date") { + test("parse date with locale") { Seq("en-US", "ru-RU").foreach { langTag => val locale = Locale.forLanguageTag(langTag) val date = new SimpleDateFormat("yyyy-MM-dd").parse("2018-11-05") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala index d0b064bb0cc54..97cc72b229c3c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala @@ -121,19 +121,17 @@ class CsvFunctionsSuite extends QueryTest with SharedSQLContext { } } - test("use locale while parsing timestamps") { + test("parse timestamps with locale") { Seq("en-US", "ko-KR", "zh-CN", "ru-RU").foreach { langTag => val locale = Locale.forLanguageTag(langTag) val ts = new SimpleDateFormat("dd/MM/yyyy HH:mm").parse("06/11/2018 18:00") val timestampFormat = "dd MMM yyyy HH:mm" val sdf = new SimpleDateFormat(timestampFormat, locale) val input = Seq(s"""${sdf.format(ts)}""").toDS() - val schema = new StructType().add("time", TimestampType) val options = Map("timestampFormat" -> timestampFormat, "locale" -> langTag) - val df = input.select(from_csv($"value", schema, options)) + val df = input.select(from_csv($"value", lit("time timestamp"), options.asJava)) - checkAnswer(df, - Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) + checkAnswer(df, Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 2b0081fee862b..ec8b6783e88c2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -582,19 +582,17 @@ class JsonFunctionsSuite extends QueryTest with SharedSQLContext { } } - test("use locale while parsing timestamps") { + test("parse timestamps with locale") { Seq("en-US", "ko-KR", "zh-CN", "ru-RU").foreach { langTag => val locale = Locale.forLanguageTag(langTag) val ts = new SimpleDateFormat("dd/MM/yyyy HH:mm").parse("06/11/2018 18:00") val timestampFormat = "dd MMM yyyy HH:mm" val sdf = new SimpleDateFormat(timestampFormat, locale) val input = Seq(s"""{"time": "${sdf.format(ts)}"}""").toDS() - val schema = new StructType().add("time", TimestampType) val options = Map("timestampFormat" -> timestampFormat, "locale" -> langTag) - val df = input.select(from_json($"value", schema, options)) + val df = input.select(from_json($"value", "time timestamp", options)) - checkAnswer(df, - Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) + checkAnswer(df, Row(Row(java.sql.Timestamp.valueOf("2018-11-06 18:00:00.0")))) } } }