Skip to content

Commit

Permalink
SPARK-24230: Add test for blocks filtered on dictionary values.
Browse files Browse the repository at this point in the history
  • Loading branch information
rdblue committed May 10, 2018
1 parent 0fa5abe commit 497bdd8
Showing 1 changed file with 12 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -879,6 +879,18 @@ class ParquetQuerySuite extends QueryTest with ParquetTest with SharedSQLContext
}
}
}

test("SPARK-24230: filter row group using dictionary") {
withSQLConf(("parquet.filter.dictionary.enabled", "true")) {
// create a table with values from 0, 2, ..., 18 that will be dictionary-encoded
withParquetTable((0 until 100).map(i => ((i * 2) % 20, s"data-$i")), "t") {
// search for a key that is not present so the dictionary filter eliminates all row groups
// Fails without SPARK-24230:
// java.io.IOException: expecting more rows but reached last block. Read 0 out of 50
checkAnswer(sql("SELECT _2 FROM t WHERE t._1 = 5"), Seq.empty)
}
}
}
}

object TestingUDT {
Expand Down

0 comments on commit 497bdd8

Please sign in to comment.