Change name to WriteInputFormatTestDataGenerator. Cleanup some var na…

…mes. Use SPARK_HOME in path for writing test sequencefile data.
pdeyhim · Jan 20, 2014 · 795a763 · 795a763
1 parent 174f520
commit 795a763
Showing 1 changed file with 16 additions and 16 deletions.
diff --git a/...rk/api/python/WriteInputFormatTests.scala → ...n/WriteInputFormatTestDataGenerator.scala b/...rk/api/python/WriteInputFormatTests.scala → ...n/WriteInputFormatTestDataGenerator.scala
@@ -5,32 +5,33 @@ import org.apache.hadoop.io._
 import scala.Array
 import java.io.{DataOutput, DataInput}
 
-case class TestWritable(var str: String, var numi: Int, var numd: Double) extends Writable {
+case class TestWritable(var str: String, var int: Int, var double: Double) extends Writable {
   def this() = this("", 0, 0.0)
 
-  def write(p1: DataOutput) = {
-    p1.writeUTF(str)
-    p1.writeInt(numi)
-    p1.writeDouble(numd)
+  def write(out: DataOutput) = {
+    out.writeUTF(str)
+    out.writeInt(int)
+    out.writeDouble(double)
   }
 
-  def readFields(p1: DataInput) = {
-    str = p1.readUTF()
-    numi = p1.readInt()
-    numd = p1.readDouble()
+  def readFields(in: DataInput) = {
+    str = in.readUTF()
+    int = in.readInt()
+    double = in.readDouble()
   }
 }
 
-object WriteInputFormatTests extends App {
+object WriteInputFormatTestDataGenerator extends App {
   import SparkContext._
 
   val sc = new SparkContext("local[2]", "test")
 
-  val textPath = "../python/test_support/data/sftext/"
-  val intPath = "../python/test_support/data/sfint/"
-  val doublePath = "../python/test_support/data/sfdouble/"
-  val arrPath = "../python/test_support/data/sfarray/"
-  val classPath = "../python/test_support/data/sfclass/"
+  val sparkHome = sys.env.get("SPARK_HOME").orElse(sys.props.get("spark.home")).get
+  val textPath = s"$sparkHome/python/test_support/data/sftext/"
+  val intPath = s"$sparkHome/python/test_support/data/sfint/"
+  val doublePath = s"$sparkHome/python/test_support/data/sfdouble/"
+  val arrPath = s"$sparkHome/python/test_support/data/sfarray/"
+  val classPath = s"$sparkHome/python/test_support/data/sfclass/"
 
   val intKeys = Seq((1.0, "aa"), (2.0, "bb"), (2.0, "aa"), (3.0, "cc"), (2.0, "bb"), (1.0, "aa"))
   sc.parallelize(intKeys).saveAsSequenceFile(intPath)
@@ -60,5 +61,4 @@ object WriteInputFormatTests extends App {
                              classOf[Text], classOf[TestWritable],
                              classOf[org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat[Text, TestWritable]])
 
-
 }