Skip to content

Commit

Permalink
rxin's comments.
Browse files Browse the repository at this point in the history
  • Loading branch information
yhuai committed Jun 19, 2014
1 parent f1a417e commit 440c5af
Show file tree
Hide file tree
Showing 6 changed files with 51 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,16 @@ case class ExplainCommand(plan: LogicalPlan) extends Command {
case class CacheCommand(tableName: String, doCache: Boolean) extends Command

/**
* Returned for the "Describe tableName" command.
* Returned for the "DESCRIBE tableName" command.
*/

/**
* Returned for the "DESCRIBE tableName" command.
* @param table The table to be described.
* @param isFormatted True if "DESCRIBE FORMATTED" is used. Otherwise, false.
* It is effective only when the table is a Hive table.
* @param isExtended True if "DESCRIBE EXTENDED" is used. Otherwise, false.
* It is effective only when the table is a Hive table.
*/
case class DescribeCommand(
table: LogicalPlan,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,10 @@ case class DescribeCommand(child: SparkPlan, output: Seq[Attribute])(
@transient context: SQLContext)
extends LeafNode with Command {

override protected[sql] lazy val sideEffectResult: Seq[(String, String, String)] =
child.output.map(field => (field.name, field.dataType.toString, None.toString))
override protected[sql] lazy val sideEffectResult: Seq[(String, String, String)] = {
Seq(("# Registered as a temporary table", null, null)) ++
child.output.map(field => (field.name, field.dataType.toString, null))
}

override def execute(): RDD[Row] = {
val rows = sideEffectResult.map {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ private[hive] object HiveQl {

protected def extractDbNameTableName(tableNameParts: Node): (Option[String], String) = {
val (db, tableName) =
tableNameParts.getChildren.map{ case Token(part, Nil) => cleanIdentifier(part)} match {
tableNameParts.getChildren.map { case Token(part, Nil) => cleanIdentifier(part) } match {
case Seq(tableOnly) => (None, tableOnly)
case Seq(databaseName, table) => (Some(databaseName), table)
}
Expand All @@ -379,7 +379,8 @@ private[hive] object HiveQl {

protected def nodeToPlan(node: Node): LogicalPlan = node match {
// Just fake explain for any of the native commands.
case Token("TOK_EXPLAIN", explainArgs) if noExplainCommands contains explainArgs.head.getText =>
case Token("TOK_EXPLAIN", explainArgs)
if noExplainCommands.contains(explainArgs.head.getText) =>
ExplainCommand(NoRelation)
case Token("TOK_EXPLAIN", explainArgs) =>
// Ignore FORMATTED if present.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.spark.sql.hive

import org.apache.spark.sql.{SQLContext}
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.planning._
import org.apache.spark.sql.catalyst.plans._
Expand Down Expand Up @@ -88,10 +88,6 @@ private[hive] trait HiveStrategies {
Seq(DescribeHiveTableCommand(
t, describe.output, describe.isFormatted, describe.isExtended)(context))
case o: LogicalPlan =>
if (describe.isFormatted)
logger.info("Formatted is ignored because it is not defined for non-Hive tables.")
if (describe.isExtended)
logger.info("Extended is ignored because it is not defined for non-Hive tables.")
Seq(DescribeCommand(planLater(o), describe.output)(context))
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ package org.apache.spark.sql.hive.execution

import org.apache.hadoop.hive.common.`type`.{HiveDecimal, HiveVarchar}
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.metastore.api.FieldSchema
import org.apache.hadoop.hive.metastore.MetaStoreUtils
import org.apache.hadoop.hive.metastore.api.FieldSchema
import org.apache.hadoop.hive.ql.Context
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils
import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Hive}
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils
import org.apache.hadoop.hive.ql.plan.{TableDesc, FileSinkDesc}
import org.apache.hadoop.hive.serde.serdeConstants
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import scala.util.Try

import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.hive.test.TestHive._
import org.apache.spark.sql.{SchemaRDD, execution, Row}
import org.apache.spark.sql.{SchemaRDD, Row}

case class TestData(a: Int, b: String)

/**
* A set of test cases expressed in Hive QL that are not covered by the tests included in the hive distribution.
Expand Down Expand Up @@ -250,11 +252,11 @@ class HiveQuerySuite extends HiveComparisonTest {
assert(Try(q0.count()).isSuccess)
}

test("Describe commands") {
hql(s"CREATE TABLE test_describe_commands (key INT, value STRING) PARTITIONED BY (dt STRING)")
test("DESCRIBE commands") {
hql(s"CREATE TABLE test_describe_commands1 (key INT, value STRING) PARTITIONED BY (dt STRING)")

hql(
"""FROM src INSERT OVERWRITE TABLE test_describe_commands PARTITION (dt='2008-06-08')
"""FROM src INSERT OVERWRITE TABLE test_describe_commands1 PARTITION (dt='2008-06-08')
|SELECT key, value
""".stripMargin)

Expand All @@ -267,20 +269,20 @@ class HiveQuerySuite extends HiveComparisonTest {
Array("# Partition Information", null, null),
Array("dt", "string", null))
) {
hql("DESCRIBE test_describe_commands")
hql("DESCRIBE test_describe_commands1")
.select('name, 'type, 'comment)
.collect()
}

// Describe a table with keyword FORMATTED
// We only
assertResult(6) {
hql("DESCRIBE FORMATTED test_describe_commands").count()
hql("DESCRIBE FORMATTED test_describe_commands1").count()
}

// Describe a table
assertResult(6) {
hql("DESCRIBE EXTENDED test_describe_commands").count()
hql("DESCRIBE EXTENDED test_describe_commands1").count()
}

// Describe a table with a fully qualified table name
Expand All @@ -292,22 +294,22 @@ class HiveQuerySuite extends HiveComparisonTest {
Array("# Partition Information", null, null),
Array("dt", "string", null))
) {
hql("DESCRIBE default.test_describe_commands")
hql("DESCRIBE default.test_describe_commands1")
.select('name, 'type, 'comment)
.collect()
}

// Describe a column is a native command
assertResult(Array(Array("value", "string", "from deserializer"))) {
hql("DESCRIBE test_describe_commands value")
hql("DESCRIBE test_describe_commands1 value")
.select('result)
.collect()
.map(_.getString(0).split("\t").map(_.trim))
}

// Describe a column is a native command
assertResult(Array(Array("value", "string", "from deserializer"))) {
hql("DESCRIBE default.test_describe_commands value")
hql("DESCRIBE default.test_describe_commands1 value")
.select('result)
.collect()
.map(_.getString(0).split("\t").map(_.trim))
Expand All @@ -325,11 +327,29 @@ class HiveQuerySuite extends HiveComparisonTest {
Array("", "", ""),
Array("dt", "string", "None"))
) {
hql("DESCRIBE test_describe_commands PARTITION (dt='2008-06-08')")
hql("DESCRIBE test_describe_commands1 PARTITION (dt='2008-06-08')")
.select('result)
.collect()
.map(_.getString(0).split("\t").map(_.trim))
}

// Describe a registered temporary table.
val testData: SchemaRDD =
TestHive.sparkContext.parallelize(
TestData(1, "str1") ::
TestData(1, "str2") :: Nil)
testData.registerAsTable("test_describe_commands2")

assertResult(
Array(
Array("# Registered as a temporary table", null, null),
Array("a", "IntegerType", null),
Array("b", "StringType", null))
) {
hql("DESCRIBE test_describe_commands2")
.select('name, 'type, 'comment)
.collect()
}
}

test("parse HQL set commands") {
Expand Down

0 comments on commit 440c5af

Please sign in to comment.