Skip to content

Commit

Permalink
fixed ClassTag in mapPartitions
Browse files Browse the repository at this point in the history
  • Loading branch information
eklavya committed Jan 23, 2014
1 parent 1442cd5 commit 60e7457
Showing 1 changed file with 9 additions and 8 deletions.
17 changes: 9 additions & 8 deletions core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
Original file line number Diff line number Diff line change
Expand Up @@ -134,27 +134,28 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
/**
* Return a new RDD by applying a function to each partition of this RDD.
*/
def mapPartitions[U](f: FlatMapFunction[java.util.Iterator[T], U], preservesPartitioning: Boolean): JavaRDD[U] = {
def fn = (x: Iterator[T]) => asScalaIterator(f.apply(asJavaIterator(x)).iterator())
JavaRDD.fromRDD(rdd.mapPartitions(fn, preservesPartitioning)(f.elementType()))(f.elementType())
}

/**
* Return a new RDD by applying a function to each partition of this RDD.
*/
def mapPartitions(f: DoubleFlatMapFunction[java.util.Iterator[T]]): JavaDoubleRDD = {
def fn = (x: Iterator[T]) => asScalaIterator(f.apply(asJavaIterator(x)).iterator())
new JavaDoubleRDD(rdd.mapPartitions(fn).map((x: java.lang.Double) => x.doubleValue()))
}

/**
* Return a new RDD by applying a function to each partition of this RDD.
* Return a new RDD by applying a function to each partition of this RDD.
*/
def mapPartitions[K2, V2](f: PairFlatMapFunction[java.util.Iterator[T], K2, V2]):
JavaPairRDD[K2, V2] = {
def fn = (x: Iterator[T]) => asScalaIterator(f.apply(asJavaIterator(x)).iterator())
JavaPairRDD.fromRDD(rdd.mapPartitions(fn))(f.keyType(), f.valueType())
}

/**
* Return a new RDD by applying a function to each partition of this RDD.
*/
def mapPartitions[U](
f: FlatMapFunction[java.util.Iterator[T], U], preservesPartitioning: Boolean): JavaRDD[U] = {
rdd.mapPartitions[U]((x => f(asJavaIterator(x)).iterator), preservesPartitioning)
}

/**
* Return a new RDD by applying a function to each partition of this RDD.
Expand Down

0 comments on commit 60e7457

Please sign in to comment.