Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Correct the test for checking pushed down filters
Browse files Browse the repository at this point in the history
HyukjinKwon committed Dec 9, 2015
1 parent f6883bb commit 3294b31
Showing 3 changed files with 19 additions and 21 deletions.
Original file line number Diff line number Diff line change
@@ -110,21 +110,6 @@ class ParquetFilterSuite extends QueryTest with ParquetTest with SharedSQLContex
checkBinaryFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df)
}

/**
* Strip Spark-side filtering in order to check if a datasource filters rows correctly.
*/
protected def stripSparkFilter(df: DataFrame): DataFrame = {
val schema = df.schema
val childRDD = df
.queryExecution
.executedPlan.asInstanceOf[org.apache.spark.sql.execution.Filter]
.child
.execute()
.map(row => Row.fromSeq(row.toSeq(schema)))

sqlContext.createDataFrame(childRDD, schema)
}

test("filter pushdown - boolean") {
withParquetDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
Original file line number Diff line number Diff line change
@@ -176,12 +176,10 @@ class JDBCSuite extends SparkFunSuite with BeforeAndAfter with SharedSQLContext
}

test("SELECT * WHERE (simple predicates)") {
assert(sql("SELECT * FROM foobar WHERE THEID < 1").collect().size === 0)
assert(sql("SELECT * FROM foobar WHERE THEID != 2").collect().size === 2)
assert(sql("SELECT * FROM foobar WHERE THEID = 1").collect().size === 1)
assert(sql("SELECT * FROM foobar WHERE NAME = 'fred'").collect().size === 1)
assert(sql("SELECT * FROM foobar WHERE NAME > 'fred'").collect().size === 2)
assert(sql("SELECT * FROM foobar WHERE NAME != 'fred'").collect().size === 2)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE THEID < 1")).collect().size === 0)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE THEID = 1")).collect().size === 1)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE NAME = 'fred'")).collect().size === 1)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE NAME > 'fred'")).collect().size === 2)
}

test("SELECT * WHERE (quoted strings)") {
Original file line number Diff line number Diff line change
@@ -179,6 +179,21 @@ private[sql] trait SQLTestUtils
try f finally sqlContext.sql(s"USE default")
}

/**
* Strip Spark-side filtering in order to check if a datasource filters rows correctly.
*/
protected def stripSparkFilter(df: DataFrame): DataFrame = {
val schema = df.schema
val childRDD = df
.queryExecution
.executedPlan.asInstanceOf[org.apache.spark.sql.execution.Filter]
.child
.execute()
.map(row => Row.fromSeq(row.toSeq(schema)))

sqlContext.createDataFrame(childRDD, schema)
}

/**
* Turn a logical plan into a [[DataFrame]]. This should be removed once we have an easier
* way to construct [[DataFrame]] directly out of local data without relying on implicits.

0 comments on commit 3294b31

Please sign in to comment.