Skip to content

Commit

Permalink
Add new constructors for SparkUI
Browse files Browse the repository at this point in the history
For two reasons - first, the existing way is ugly because we have to instantiate the SparkUI
by calling new SparkUI(null). Second, this provides a way to configure the persisted port
through SparkConf.
  • Loading branch information
andrewor14 committed Mar 5, 2014
1 parent 18b256d commit e375431
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 18 deletions.
24 changes: 9 additions & 15 deletions core/src/main/scala/org/apache/spark/ui/SparkUI.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package org.apache.spark.ui

import org.eclipse.jetty.server.{Handler, Server}

import org.apache.spark.{Logging, SparkContext, SparkEnv}
import org.apache.spark.{SparkConf, Logging, SparkContext, SparkEnv}
import org.apache.spark.scheduler.{SparkReplayerBus, EventLoggingListener}
import org.apache.spark.ui.JettyUtils._
import org.apache.spark.ui.env.EnvironmentUI
Expand All @@ -29,15 +29,19 @@ import org.apache.spark.ui.storage.BlockManagerUI
import org.apache.spark.util.Utils

/** Top level user interface for Spark. */
private[spark] class SparkUI(val sc: SparkContext) extends Logging {
private[spark] class SparkUI(val sc: SparkContext, conf: SparkConf) extends Logging {

def this() = this(null, new SparkConf())
def this(conf: SparkConf) = this(null, conf)
def this(sc: SparkContext) = this(sc, sc.conf)

// If SparkContext is not provided, assume this UI is rendered from persisted storage
val live = sc != null
val host = Option(System.getenv("SPARK_PUBLIC_DNS")).getOrElse(Utils.localHostName())
var port = if (live) {
sc.conf.get("spark.ui.port", SparkUI.DEFAULT_PORT).toInt
conf.get("spark.ui.port", SparkUI.DEFAULT_PORT).toInt
} else {
SparkUI.DEFAULT_PERSISTED_PORT.toInt
conf.get("spark.persisted.ui.port", SparkUI.DEFAULT_PERSISTED_PORT).toInt
}
var boundPort: Option[Int] = None
var server: Option[Server] = None
Expand Down Expand Up @@ -74,16 +78,6 @@ private[spark] class SparkUI(val sc: SparkContext) extends Logging {
// Only replay events if this SparkUI is not live
private var replayerBus: Option[SparkReplayerBus] = None

// Only meaningful if port is set before binding
def setPort(p: Int) = {
if (boundPort.isDefined) {
logWarning("Attempted to set Spark Web UI port after it is already bound to %s."
.format(appUIAddress))
} else {
port = p
}
}

def setAppName(name: String) = appName = name

/** Bind the HTTP server which backs this web interface */
Expand Down Expand Up @@ -113,7 +107,7 @@ private[spark] class SparkUI(val sc: SparkContext) extends Logging {

// Listen for events from the SparkContext if it exists, otherwise from persisted storage
val eventBus = if (live) {
eventLogger = Some(new EventLoggingListener(sc.appName, sc.conf))
eventLogger = Some(new EventLoggingListener(sc.appName, conf))
sc.listenerBus.addListener(eventLogger.get)
sc.listenerBus
} else {
Expand Down
11 changes: 8 additions & 3 deletions core/src/main/scala/org/apache/spark/ui/UIReloader.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,22 @@

package org.apache.spark.ui

import org.apache.spark.SparkConf

/**
* Reload a persisted UI independently from a SparkContext
* A simple example that reloads a persisted UI independently from a SparkContext
*/
object UIReloader {
def main(args: Array[String]) {
if (args.length < 1) {
println("Usage: ./bin/spark-class org.apache.spark.ui.UIReloader [log path]")
println("Usage: ./bin/spark-class org.apache.spark.ui.UIReloader [log path] [port]")
System.exit(1)
}

val ui = new SparkUI(null)
val port = if (args.length == 2) args(1) else "14040"
val conf = new SparkConf()
conf.set("spark.persisted.ui.port", port)
val ui = new SparkUI(conf)
ui.bind()
ui.start()
val success = ui.renderFromPersistedStorage(args(0))
Expand Down

0 comments on commit e375431

Please sign in to comment.