diff --git a/README.md b/README.md index 004b229bb96fe..fb7b0d5aeef6e 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ $ sbt/sbt > test:compile [info] Compiling 10 Scala sources to catalyst/target/scala-2.10/test-classes... [success] Total time: 15 s, completed Dec 20, 2013 12:00:06 PM -> test-only catalyst.shark2.DslQueryTests +> test-only catalyst.execution.DslQueryTests ``` Any command that is prefixed with a `~` (e.g. `~compile`) will be run automatically in a loop each time any dependent files have changed. @@ -54,13 +54,13 @@ import catalyst.plans.logical._ import catalyst.rules._ import catalyst.types._ import catalyst.util._ -import catalyst.shark2.TestShark._ +import catalyst.execution.TestShark._ Welcome to Scala version 2.10.3 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_45). Type in expressions to have them evaluated. Type :help for more information. scala> val query = "SELECT * FROM (SELECT * FROM src) a".q -query: catalyst.shark2.TestShark.SharkSqlQuery = +query: catalyst.execution.TestShark.SharkSqlQuery = SELECT * FROM (SELECT * FROM src) a == Logical Plan == Project {key#0,value#1} diff --git a/build.sbt b/build.sbt index 20183c04d0e31..4a180507d74e1 100644 --- a/build.sbt +++ b/build.sbt @@ -45,7 +45,7 @@ import catalyst.plans.logical._ import catalyst.rules._ import catalyst.types._ import catalyst.util._ -import catalyst.shark2.TestShark._""" +import catalyst.execution.TestShark._""" site.settings diff --git a/src/main/scala/catalyst/examples/SchemaRddExample.scala b/src/main/scala/catalyst/examples/SchemaRddExample.scala index 89a64bd81b11f..f89aae50925bb 100644 --- a/src/main/scala/catalyst/examples/SchemaRddExample.scala +++ b/src/main/scala/catalyst/examples/SchemaRddExample.scala @@ -3,7 +3,7 @@ package examples import plans.logical.LocalRelation -import shark2.TestShark._ +import execution.TestShark._ import dsl._ object SchemaRddExample { diff --git a/src/main/scala/catalyst/examples/ViewsExample.scala b/src/main/scala/catalyst/examples/ViewsExample.scala index a6049146eb692..7eee3b868f1f8 100644 --- a/src/main/scala/catalyst/examples/ViewsExample.scala +++ b/src/main/scala/catalyst/examples/ViewsExample.scala @@ -6,7 +6,7 @@ import catalyst.plans.logical._ /* Implicit Conversions */ import dsl._ -import shark2.TestShark._ // For .toRdd execution using locally running test Shark. +import execution.TestShark._ // For .toRdd execution using locally running test Shark. object ViewsExample { def main(args: Array[String]): Unit = { diff --git a/src/main/scala/catalyst/shark2/FunctionRegistry.scala b/src/main/scala/catalyst/execution/FunctionRegistry.scala similarity index 99% rename from src/main/scala/catalyst/shark2/FunctionRegistry.scala rename to src/main/scala/catalyst/execution/FunctionRegistry.scala index cc35674a29d4e..691fb0abc06d4 100644 --- a/src/main/scala/catalyst/shark2/FunctionRegistry.scala +++ b/src/main/scala/catalyst/execution/FunctionRegistry.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.apache.hadoop.hive.ql.exec.{FunctionInfo, FunctionRegistry} diff --git a/src/main/scala/catalyst/shark2/MetastoreCatalog.scala b/src/main/scala/catalyst/execution/MetastoreCatalog.scala similarity index 99% rename from src/main/scala/catalyst/shark2/MetastoreCatalog.scala rename to src/main/scala/catalyst/execution/MetastoreCatalog.scala index 6dcd2b9d696e9..06b62687e88d6 100644 --- a/src/main/scala/catalyst/shark2/MetastoreCatalog.scala +++ b/src/main/scala/catalyst/execution/MetastoreCatalog.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.metastore.api.{FieldSchema, Partition, Table, StorageDescriptor, SerDeInfo} diff --git a/src/main/scala/catalyst/shark2/SharkInstance.scala b/src/main/scala/catalyst/execution/SharkInstance.scala similarity index 98% rename from src/main/scala/catalyst/shark2/SharkInstance.scala rename to src/main/scala/catalyst/execution/SharkInstance.scala index 871d470210acc..edd7eb93b4431 100644 --- a/src/main/scala/catalyst/shark2/SharkInstance.scala +++ b/src/main/scala/catalyst/execution/SharkInstance.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import java.io.File @@ -40,7 +40,7 @@ abstract class SharkInstance extends Logging { lazy val sc = createContext() protected def createContext() = { - SharkEnv.initWithSharkContext("catalyst.shark2", master) + SharkEnv.initWithSharkContext("catalyst.execution", master) } /** Sets up the system initially or after a RESET command */ diff --git a/src/main/scala/catalyst/shark2/SharkPlan.scala b/src/main/scala/catalyst/execution/SharkPlan.scala similarity index 97% rename from src/main/scala/catalyst/shark2/SharkPlan.scala rename to src/main/scala/catalyst/execution/SharkPlan.scala index 51b706ead1df0..519336f217fb1 100644 --- a/src/main/scala/catalyst/shark2/SharkPlan.scala +++ b/src/main/scala/catalyst/execution/SharkPlan.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import catalyst.plans.QueryPlan import org.apache.spark.rdd.RDD diff --git a/src/main/scala/catalyst/shark2/TestShark.scala b/src/main/scala/catalyst/execution/TestShark.scala similarity index 99% rename from src/main/scala/catalyst/shark2/TestShark.scala rename to src/main/scala/catalyst/execution/TestShark.scala index d3d08f1345a5a..e500d4c79c080 100644 --- a/src/main/scala/catalyst/shark2/TestShark.scala +++ b/src/main/scala/catalyst/execution/TestShark.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import java.io.File import scala.collection.mutable @@ -56,7 +56,7 @@ object TestShark extends SharkInstance { System.clearProperty("spark.driver.port") System.clearProperty("spark.hostPort") - SharkEnv.initWithSharkContext("catalyst.shark2.TestShark", master) + SharkEnv.initWithSharkContext("catalyst.execution.TestShark", master) } /** The location of the compiled hive distribution */ diff --git a/src/main/scala/catalyst/shark2/Transform.scala b/src/main/scala/catalyst/execution/Transform.scala similarity index 98% rename from src/main/scala/catalyst/shark2/Transform.scala rename to src/main/scala/catalyst/execution/Transform.scala index d271e8146abeb..0178efbc8c9dd 100644 --- a/src/main/scala/catalyst/shark2/Transform.scala +++ b/src/main/scala/catalyst/execution/Transform.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import catalyst.expressions._ import shark.SharkContext diff --git a/src/main/scala/catalyst/shark2/aggregates.scala b/src/main/scala/catalyst/execution/aggregates.scala similarity index 99% rename from src/main/scala/catalyst/shark2/aggregates.scala rename to src/main/scala/catalyst/execution/aggregates.scala index 33d6ce69b4a6d..dfdb6e68fee31 100644 --- a/src/main/scala/catalyst/shark2/aggregates.scala +++ b/src/main/scala/catalyst/execution/aggregates.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import catalyst.errors._ import catalyst.expressions._ diff --git a/src/main/scala/catalyst/shark2/basicOperators.scala b/src/main/scala/catalyst/execution/basicOperators.scala similarity index 99% rename from src/main/scala/catalyst/shark2/basicOperators.scala rename to src/main/scala/catalyst/execution/basicOperators.scala index d61d5e1f2cc61..a4ea279240041 100644 --- a/src/main/scala/catalyst/shark2/basicOperators.scala +++ b/src/main/scala/catalyst/execution/basicOperators.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.apache.hadoop.hive.ql.plan.{FileSinkDesc, TableDesc} import org.apache.hadoop.hive.serde2.objectinspector.{PrimitiveObjectInspector, StructObjectInspector} diff --git a/src/main/scala/catalyst/shark2/debug.scala b/src/main/scala/catalyst/execution/debug.scala similarity index 97% rename from src/main/scala/catalyst/shark2/debug.scala rename to src/main/scala/catalyst/execution/debug.scala index e0e91a840d66e..a1e262f3e3035 100644 --- a/src/main/scala/catalyst/shark2/debug.scala +++ b/src/main/scala/catalyst/execution/debug.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution object DebugQuery { def apply(plan: SharkPlan): SharkPlan = { diff --git a/src/main/scala/catalyst/shark2/hiveOperators.scala b/src/main/scala/catalyst/execution/hiveOperators.scala similarity index 99% rename from src/main/scala/catalyst/shark2/hiveOperators.scala rename to src/main/scala/catalyst/execution/hiveOperators.scala index d39d7fc86ee43..fb9e94074dd43 100644 --- a/src/main/scala/catalyst/shark2/hiveOperators.scala +++ b/src/main/scala/catalyst/execution/hiveOperators.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils diff --git a/src/main/scala/catalyst/shark2/joins.scala b/src/main/scala/catalyst/execution/joins.scala similarity index 99% rename from src/main/scala/catalyst/shark2/joins.scala rename to src/main/scala/catalyst/execution/joins.scala index e5084b6c71284..706d52d1fea93 100644 --- a/src/main/scala/catalyst/shark2/joins.scala +++ b/src/main/scala/catalyst/execution/joins.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.apache.spark.rdd.RDD diff --git a/src/main/scala/catalyst/shark2/package.scala b/src/main/scala/catalyst/execution/package.scala similarity index 97% rename from src/main/scala/catalyst/shark2/package.scala rename to src/main/scala/catalyst/execution/package.scala index e0857cd74d4c5..6e366263d3511 100644 --- a/src/main/scala/catalyst/shark2/package.scala +++ b/src/main/scala/catalyst/execution/package.scala @@ -13,7 +13,7 @@ import types._ * Currently functions that are not supported by this implementation are passed back to the original Shark * implementation for execution. */ -package object shark2 { +package object execution { type Row = catalyst.expressions.Row implicit class typeInfoConversions(dt: DataType) { diff --git a/src/main/scala/catalyst/shark2/planningStrategies.scala b/src/main/scala/catalyst/execution/planningStrategies.scala similarity index 77% rename from src/main/scala/catalyst/shark2/planningStrategies.scala rename to src/main/scala/catalyst/execution/planningStrategies.scala index c368eb90ef3a2..556503b6fbbdc 100644 --- a/src/main/scala/catalyst/shark2/planningStrategies.scala +++ b/src/main/scala/catalyst/execution/planningStrategies.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import shark.SharkContext @@ -25,9 +25,9 @@ abstract trait PlanningStrategies { def apply(plan: LogicalPlan): Seq[SharkPlan] = plan match { // Push attributes into table scan when possible. case p @ logical.Project(projectList, m: MetastoreRelation) if isSimpleProject(projectList) => - shark2.HiveTableScan(projectList.asInstanceOf[Seq[Attribute]], m) :: Nil + execution.HiveTableScan(projectList.asInstanceOf[Seq[Attribute]], m) :: Nil case m: MetastoreRelation => - shark2.HiveTableScan(m.output, m) :: Nil + execution.HiveTableScan(m.output, m) :: Nil case _ => Nil } @@ -59,7 +59,7 @@ abstract trait PlanningStrategies { def apply(plan: LogicalPlan): Seq[SharkPlan] = plan match { case logical.Aggregate(Nil, agg, child) if onlyAllowedAggregates(agg) => - shark2.SparkAggregate(agg, planLater(child))(sc) :: Nil + execution.SparkAggregate(agg, planLater(child))(sc) :: Nil case _ => Nil } } @@ -87,11 +87,11 @@ abstract trait PlanningStrategies { val leftKeys = joinKeys.map(_._1) val rightKeys = joinKeys.map(_._2) - val joinOp = shark2.SparkEquiInnerJoin(leftKeys, rightKeys, planLater(left), planLater(right)) + val joinOp = execution.SparkEquiInnerJoin(leftKeys, rightKeys, planLater(left), planLater(right)) // Make sure other conditions are met if present. if(otherPredicates.nonEmpty) - shark2.Filter(combineConjunctivePredicates(otherPredicates), joinOp) :: Nil + execution.Filter(combineConjunctivePredicates(otherPredicates), joinOp) :: Nil else joinOp :: Nil } else { @@ -112,17 +112,17 @@ abstract trait PlanningStrategies { object BroadcastNestedLoopJoin extends Strategy { def apply(plan: LogicalPlan): Seq[SharkPlan] = plan match { case logical.Join(left, right, joinType, condition) => - shark2.BroadcastNestedLoopJoin(planLater(left), planLater(right), joinType, condition)(sc) :: Nil + execution.BroadcastNestedLoopJoin(planLater(left), planLater(right), joinType, condition)(sc) :: Nil case _ => Nil } } object CartesianProduct extends Strategy { def apply(plan: LogicalPlan): Seq[SharkPlan] = plan match { - case logical.Join(left, right, _, None) => shark2.CartesianProduct(planLater(left), planLater(right)) :: Nil + case logical.Join(left, right, _, None) => execution.CartesianProduct(planLater(left), planLater(right)) :: Nil case logical.Join(left, right, Inner, Some(condition)) => - shark2.Filter(condition, - shark2.CartesianProduct(planLater(left), planLater(right))) :: Nil + execution.Filter(condition, + execution.CartesianProduct(planLater(left), planLater(right))) :: Nil case _ => Nil } } @@ -131,21 +131,21 @@ abstract trait PlanningStrategies { object BasicOperators extends Strategy { def apply(plan: LogicalPlan): Seq[SharkPlan] = plan match { case logical.Sort(sortExprs, child) => - shark2.Sort(sortExprs, planLater(child)) :: Nil + execution.Sort(sortExprs, planLater(child)) :: Nil case logical.Project(projectList, child) => - shark2.Project(projectList, planLater(child)) :: Nil + execution.Project(projectList, planLater(child)) :: Nil case logical.Filter(condition, child) => - shark2.Filter(condition, planLater(child)) :: Nil + execution.Filter(condition, planLater(child)) :: Nil case logical.Aggregate(group, agg, child) => - shark2.Aggregate(group, agg, planLater(child)) :: Nil + execution.Aggregate(group, agg, planLater(child)) :: Nil case logical.LocalRelation(output, data) => - shark2.LocalRelation(output, data.map(_.productIterator.toVector))(sc) :: Nil + execution.LocalRelation(output, data.map(_.productIterator.toVector))(sc) :: Nil case logical.StopAfter(limit, child) => - shark2.StopAfter(Evaluate(limit, Nil).asInstanceOf[Int], planLater(child))(sc) :: Nil + execution.StopAfter(Evaluate(limit, Nil).asInstanceOf[Int], planLater(child))(sc) :: Nil case logical.Union(left, right) => - shark2.Union(planLater(left), planLater(right))(sc) :: Nil + execution.Union(planLater(left), planLater(right))(sc) :: Nil case logical.Transform(input, script, output, child) => - shark2.Transform(input, script, output, planLater(child))(sc) :: Nil + execution.Transform(input, script, output, planLater(child))(sc) :: Nil case _ => Nil } } diff --git a/src/main/scala/catalyst/expressions/BoundAttribute.scala b/src/main/scala/catalyst/expressions/BoundAttribute.scala index f03c39141b1bd..fa2ef0475caae 100644 --- a/src/main/scala/catalyst/expressions/BoundAttribute.scala +++ b/src/main/scala/catalyst/expressions/BoundAttribute.scala @@ -4,7 +4,7 @@ package expressions import rules._ import errors._ -import shark2.SharkPlan +import execution.SharkPlan /** * A bound reference points to a specific slot in the input tuple, allowing the actual value to be retrieved more diff --git a/src/test/scala/catalyst/shark2/BigDataBenchmark.scala b/src/test/scala/catalyst/execution/BigDataBenchmark.scala similarity index 99% rename from src/test/scala/catalyst/shark2/BigDataBenchmark.scala rename to src/test/scala/catalyst/execution/BigDataBenchmark.scala index 20322169995c5..d5e05ba28bacc 100644 --- a/src/test/scala/catalyst/shark2/BigDataBenchmark.scala +++ b/src/test/scala/catalyst/execution/BigDataBenchmark.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import java.io.File diff --git a/src/test/scala/catalyst/shark2/DslQueryTests.scala b/src/test/scala/catalyst/execution/DslQueryTests.scala similarity index 99% rename from src/test/scala/catalyst/shark2/DslQueryTests.scala rename to src/test/scala/catalyst/execution/DslQueryTests.scala index 146745deca780..0e8463996ad54 100644 --- a/src/test/scala/catalyst/shark2/DslQueryTests.scala +++ b/src/test/scala/catalyst/execution/DslQueryTests.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import org.scalatest.{BeforeAndAfterAll, FunSuite} diff --git a/src/test/scala/catalyst/shark2/HiveAstTransformSuite.scala b/src/test/scala/catalyst/execution/HiveAstTransformSuite.scala similarity index 100% rename from src/test/scala/catalyst/shark2/HiveAstTransformSuite.scala rename to src/test/scala/catalyst/execution/HiveAstTransformSuite.scala diff --git a/src/test/scala/catalyst/shark2/HiveComparisionTest.scala b/src/test/scala/catalyst/execution/HiveComparisionTest.scala similarity index 99% rename from src/test/scala/catalyst/shark2/HiveComparisionTest.scala rename to src/test/scala/catalyst/execution/HiveComparisionTest.scala index 273a7207752e2..203a5c22b5919 100644 --- a/src/test/scala/catalyst/shark2/HiveComparisionTest.scala +++ b/src/test/scala/catalyst/execution/HiveComparisionTest.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import shark.{SharkContext, SharkEnv} diff --git a/src/test/scala/catalyst/shark2/HiveCompatability.scala b/src/test/scala/catalyst/execution/HiveCompatability.scala similarity index 99% rename from src/test/scala/catalyst/shark2/HiveCompatability.scala rename to src/test/scala/catalyst/execution/HiveCompatability.scala index 9c044592a9d3f..c72e655932ebc 100644 --- a/src/test/scala/catalyst/shark2/HiveCompatability.scala +++ b/src/test/scala/catalyst/execution/HiveCompatability.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import java.io._ diff --git a/src/test/scala/catalyst/shark2/HiveQueryFileTest.scala b/src/test/scala/catalyst/execution/HiveQueryFileTest.scala similarity index 98% rename from src/test/scala/catalyst/shark2/HiveQueryFileTest.scala rename to src/test/scala/catalyst/execution/HiveQueryFileTest.scala index 2cc4832a46391..fa93af8daa3c6 100644 --- a/src/test/scala/catalyst/shark2/HiveQueryFileTest.scala +++ b/src/test/scala/catalyst/execution/HiveQueryFileTest.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution import java.io._ diff --git a/src/test/scala/catalyst/shark2/HiveQueryTests.scala b/src/test/scala/catalyst/execution/HiveQueryTests.scala similarity index 99% rename from src/test/scala/catalyst/shark2/HiveQueryTests.scala rename to src/test/scala/catalyst/execution/HiveQueryTests.scala index c2d68588d31d8..ae710ea939adc 100644 --- a/src/test/scala/catalyst/shark2/HiveQueryTests.scala +++ b/src/test/scala/catalyst/execution/HiveQueryTests.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution /** * A set of test cases expressed in Hive QL that are not covered by the tests included in the hive distribution. diff --git a/src/test/scala/catalyst/shark2/HiveResolutionSuite.scala b/src/test/scala/catalyst/execution/HiveResolutionSuite.scala similarity index 98% rename from src/test/scala/catalyst/shark2/HiveResolutionSuite.scala rename to src/test/scala/catalyst/execution/HiveResolutionSuite.scala index ce9f1aadb8771..13dfb951cb55a 100644 --- a/src/test/scala/catalyst/shark2/HiveResolutionSuite.scala +++ b/src/test/scala/catalyst/execution/HiveResolutionSuite.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution /** * A set of test cases expressed in Hive QL that are not covered by the tests included in the hive distribution. diff --git a/src/test/scala/catalyst/shark2/HiveTypeCoersionSuite.scala b/src/test/scala/catalyst/execution/HiveTypeCoersionSuite.scala similarity index 95% rename from src/test/scala/catalyst/shark2/HiveTypeCoersionSuite.scala rename to src/test/scala/catalyst/execution/HiveTypeCoersionSuite.scala index 0fb3d13bd20b7..095dce23aade9 100644 --- a/src/test/scala/catalyst/shark2/HiveTypeCoersionSuite.scala +++ b/src/test/scala/catalyst/execution/HiveTypeCoersionSuite.scala @@ -1,5 +1,5 @@ package catalyst -package shark2 +package execution /** * A set of tests that validate type promotion rules.