Skip to content

Commit

Permalink
address aaron's comments
Browse files Browse the repository at this point in the history
  • Loading branch information
haoyuan committed Apr 2, 2014
1 parent d9a6438 commit 9b97935
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 9 deletions.
4 changes: 2 additions & 2 deletions core/src/main/scala/org/apache/spark/SparkContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ package org.apache.spark

import java.io._
import java.net.URI
import java.util.{Properties, UUID}
import java.util.concurrent.atomic.AtomicInteger
import java.util.{Properties, UUID}
import java.util.UUID.randomUUID
import scala.collection.{Map, Set}
import scala.collection.generic.Growable
Expand Down Expand Up @@ -125,7 +125,7 @@ class SparkContext(

val master = conf.get("spark.master")
val appName = conf.get("spark.app.name")

// Generate the random name for a temp folder in Tachyon
// Add a timestamp as the suffix here to make it more safe
val tachyonFolderName = "spark-" + randomUUID.toString()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ import org.apache.spark.util.Utils
* @param rootDirs The directories to use for storing block files. Data will be hashed among these.
*/
private[spark] class TachyonBlockManager(
shuffleManager: ShuffleBlockManager,
rootDirs: String,
shuffleManager: ShuffleBlockManager,
rootDirs: String,
val master: String)
extends Logging {

Expand All @@ -49,21 +49,21 @@ private[spark] class TachyonBlockManager(
}

private val MAX_DIR_CREATION_ATTEMPTS = 10
private val subDirsPerTachyonDir =
private val subDirsPerTachyonDir =
shuffleManager.conf.get("spark.tachyonStore.subDirectories", "64").toInt

// Create one Tachyon directory for each path mentioned in spark.tachyonStore.folderName.dir;
// then, inside this directory, create multiple subdirectories that we will hash files into,
// Create one Tachyon directory for each path mentioned in spark.tachyonStore.folderName;
// then, inside this directory, create multiple subdirectories that we will hash files into,
// in order to avoid having really large inodes at the top level in Tachyon.
private val tachyonDirs: Array[TachyonFile] = createTachyonDirs()
private val subDirs = Array.fill(tachyonDirs.length)(new Array[TachyonFile](subDirsPerTachyonDir))

addShutdownHook()

def removeFile(file: TachyonFile): Boolean = {
client.delete(file.getPath(), false)
}

def fileExists(file: TachyonFile): Boolean = {
client.exist(file.getPath())
}
Expand Down

0 comments on commit 9b97935

Please sign in to comment.