From 8f047202a15a373c92724d99ec2e4ab2d7d30a07 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 2 Jun 2016 15:15:12 -0700 Subject: [PATCH] Fix bug. --- .../org/apache/spark/storage/BlockManager.scala | 13 ++++++++----- .../apache/spark/storage/BlockManagerSuite.scala | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 288f756bca39b..339ee1442e158 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -507,11 +507,14 @@ private[spark] class BlockManager( // Look for block on disk, potentially storing it back in memory if required if (level.useDisk) { logDebug(s"Getting block $blockId from disk") - val bytes: ByteBuffer = diskStore.getBytes(blockId) match { - case Some(b) => b - case None => - throw new BlockException( - blockId, s"Block $blockId not found on disk, though it should be") + val bytes: ByteBuffer = if (diskStore.contains(blockId)) { + // DiskStore.getBytes() always returns Some, so this .get() is guaranteed to be safe + diskStore.getBytes(blockId).get + } else { + // Remove the missing block so that its unavailability is reported to the driver + removeBlock(blockId) + throw new BlockException( + blockId, s"Block $blockId not found on disk, though it should be") } assert(0 == bytes.position()) diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala index 82d3c4964da86..47e854596f5dc 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala @@ -1375,7 +1375,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE // Because the BlockManager's metadata claims that the block exists (i.e. that it's present // in at least one store), the read attempts to read it and fails when the on-disk file is // missing. - intercept[SparkException] { + intercept[BlockException] { readMethod(store) } // Subsequent read attempts will succeed; the block isn't present but we return an expected