Skip to content

Commit

Permalink
Consider the number of selector too
Browse files Browse the repository at this point in the history
  • Loading branch information
zsxwing committed Mar 11, 2016
1 parent ea99768 commit 8422f6e
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 8 deletions.
20 changes: 13 additions & 7 deletions core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -270,19 +270,25 @@ private[spark] object JettyUtils extends Logging {

gzipHandlers.foreach(collection.addHandler)
connectors.foreach(_.setHost(hostName))
// As each Acceptor will use one thread, the number of threads should at least be the number
// of acceptors plus 1. (See SPARK-13776)
// As each acceptor and each selector will use one thread, the number of threads should at
// least be the number of acceptors and selectors plus 1. (See SPARK-13776)
var minThreads = 1
connectors.collect { case c: AbstractConnector => c }.foreach { c =>
connectors.foreach { c =>
// Currently we only use "SelectChannelConnector"
val connector = c.asInstanceOf[SelectChannelConnector]
// Limit the max acceptor number to 8 so that we don't waste a lot of threads
c.setAcceptors(Math.min(c.getAcceptors, 8))
minThreads += c.getAcceptors
connector.setAcceptors(math.min(connector.getAcceptors, 8))
// The number of selectors always equals to the number of acceptors
minThreads += connector.getAcceptors * 2
}
server.setConnectors(connectors.toArray)

val pool = new QueuedThreadPool
pool.setMaxThreads(Math.max(pool.getMaxThreads, minThreads))
pool.setMinThreads(Math.min(pool.getMinThreads, pool.getMaxThreads))
if (serverName.nonEmpty) {
pool.setName(serverName)
}
pool.setMaxThreads(math.max(pool.getMaxThreads, minThreads))
pool.setMinThreads(math.min(pool.getMinThreads, pool.getMaxThreads))
pool.setDaemon(true)
server.setThreadPool(pool)
val errorHandler = new ErrorHandler()
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/scala/org/apache/spark/ui/WebUI.scala
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ private[spark] abstract class WebUI(
def bind() {
assert(!serverInfo.isDefined, "Attempted to bind %s more than once!".format(className))
try {
var host = Option(conf.getenv("SPARK_LOCAL_IP")).getOrElse("0.0.0.0")
val host = Option(conf.getenv("SPARK_LOCAL_IP")).getOrElse("0.0.0.0")
serverInfo = Some(startJettyServer(host, port, sslOptions, handlers, conf, name))
logInfo("Bound %s to %s, and started at http://%s:%d".format(className, host,
publicHostName, boundPort))
Expand Down

0 comments on commit 8422f6e

Please sign in to comment.