From a1e1b578cd8f7aa45fd0db107b194c500284ae79 Mon Sep 17 00:00:00 2001 From: petermaxlee Date: Wed, 3 Aug 2016 13:17:32 -0700 Subject: [PATCH] Use a more concise and human readable format. --- .../sql-tests/results/number-format.sql.out | 50 +++++++++++++ .../sql-tests/results/number-format.sql.xml | 49 ------------- .../apache/spark/sql/SQLQueryTestSuite.scala | 73 ++++++++++++------- 3 files changed, 95 insertions(+), 77 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/results/number-format.sql.out delete mode 100644 sql/core/src/test/resources/sql-tests/results/number-format.sql.xml diff --git a/sql/core/src/test/resources/sql-tests/results/number-format.sql.out b/sql/core/src/test/resources/sql-tests/results/number-format.sql.out new file mode 100644 index 0000000000000..8757119f8f6c3 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/number-format.sql.out @@ -0,0 +1,50 @@ +-- Automatically generated by org.apache.spark.sql.SQLQueryTestSuite +-- Number of queries: 4 + + +-- !query 0 +select 1, -1 +-- !query 0 schema +int, int +-- !query 0 output ++---+----+ +| 1|(-1)| ++---+----+ +| 1| -1| ++---+----+ + + +-- !query 1 +select 2147483648, -2147483649 +-- !query 1 schema +bigint, bigint +-- !query 1 output ++----------+-------------+ +|2147483648|(-2147483649)| ++----------+-------------+ +|2147483648| -2147483649| ++----------+-------------+ + + +-- !query 2 +select 9223372036854775808, -9223372036854775809 +-- !query 2 schema +decimal(19,0), decimal(19,0) +-- !query 2 output ++-------------------+----------------------+ +|9223372036854775808|(-9223372036854775809)| ++-------------------+----------------------+ +|9223372036854775808| -9223372036854775809| ++-------------------+----------------------+ + + +-- !query 3 +select 0.3, -0.8, .5, -.18 +-- !query 3 schema +decimal(1,1), decimal(1,1), decimal(1,1), decimal(2,2) +-- !query 3 output ++---+------+---+-------+ +|0.3|(-0.8)|0.5|(-0.18)| ++---+------+---+-------+ +|0.3| -0.8|0.5| -0.18| ++---+------+---+-------+ diff --git a/sql/core/src/test/resources/sql-tests/results/number-format.sql.xml b/sql/core/src/test/resources/sql-tests/results/number-format.sql.xml deleted file mode 100644 index f7f9f9ae13ec0..0000000000000 --- a/sql/core/src/test/resources/sql-tests/results/number-format.sql.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index b52fb737fe9c8..3238a97c7dd5e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -42,11 +42,28 @@ import org.apache.spark.sql.test.SharedSQLContext * For example: * {{{ * -- this is a comment - * select 1 + 2; + * select 1, -1; * select current_date; * }}} * - * Result files are encoded as XMLs. + * The format for golden result files look roughly like: + * {{{ + * -- some header information + * + * -- !query 0 + * select 1, -1 + * -- !query 0 schema + * int, int + * -- !query 0 output + * +---+----+ + * | 1|(-1)| + * +---+----+ + * | 1| -1| + * +---+----+ + * + * -- !query 1 + * ... + * }}} */ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { @@ -68,15 +85,14 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { /** A single SQL query's output. */ private case class QueryOutput(sql: String, schema: String, output: String) { - def toXML: String = { - // We are explicitly not using multi-line string due to stripMargin removing |s, - // and not using XML interpolation because there is no simple way to indent outputs nicely - // (scala.xml.PrettyPrinter has issue with tabs). - "\n" + - s" \n" + - s" \n" + - s" \n" + - s"" + def toString(queryIndex: Int): String = { + // We are explicitly not using multi-line string due to stripMargin removing "|" in output. + s"-- !query $queryIndex\n" + + sql + "\n" + + s"-- !query $queryIndex schema\n" + + schema + "\n" + + s"-- !query $queryIndex output\n" + + output } } @@ -96,7 +112,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // List of SQL queries to run val queries: Seq[String] = { - val cleaned = input.split("\n").filterNot(_.matches("--.*(?<=[^\\\\]);")).mkString("\n") + // val cleaned = input.split("\n").filterNot(_.matches("--.*(?<=[^\\\\]);")).mkString("\n") + val cleaned = input.split("\n").filterNot(_.startsWith("--")).mkString("\n") + // note: this is not a robust way to split queries using semicolon, but works for now. cleaned.split("(?<=[^\\\\]);").map(_.trim).filterNot(q => q == "").toSeq } @@ -111,27 +129,26 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { } if (regenerateGoldenFiles) { - // If generate golden file flag is on, create the golden file. - // Again, we are explicitly not using multi-line string due to stripMargin removing |s, - // and not using XML interpolation because there is no simple way to indent outputs nicely - // (scala.xml.PrettyPrinter has issue with tabs). - val xmlOutput = { - "\n" + - "\n" + - outputs.map(_.toXML).mkString("\n") + - "\n\n" + // Again, we are explicitly not using multi-line string due to stripMargin removing "|". + val goldenOutput = { + s"-- Automatically generated by ${getClass.getName}\n" + + s"-- Number of queries: ${outputs.size}\n\n\n" + + outputs.zipWithIndex.map{case (qr, i) => qr.toString(i)}.mkString("\n\n\n") + "\n" } - stringToFile(new File(testCase.resultFile), xmlOutput) + stringToFile(new File(testCase.resultFile), goldenOutput) } // Read back the golden file. val expectedOutputs: Seq[QueryOutput] = { - val xml = scala.xml.XML.loadString(fileToString(new File(testCase.resultFile))) - (xml \ "query").map { q => + val goldenOutput = fileToString(new File(testCase.resultFile)) + val segments = goldenOutput.split("-- !query.+\n") + assert(segments.size == outputs.size * 3 + 1) // each query has 3 segments, plus the header + Seq.tabulate(outputs.size) { i => QueryOutput( - sql = (q \ "sql").text, - schema = (q \ "schema").text, - output = (q \ "output").text.trim) + sql = segments(i * 3 + 1).trim, + schema = segments(i * 3 + 2).trim, + output = segments(i * 3 + 3).trim + ) } } @@ -149,7 +166,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { private def listTestCases(): Seq[TestCase] = { listFilesRecursively(new File(inputFilePath)).map { file => - val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".xml" + val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" TestCase(file.getName, file.getAbsolutePath, resultFile) } }