Skip to content

Commit

Permalink
History Server: updated order for multiple attempts
Browse files Browse the repository at this point in the history
  • Loading branch information
@rekhajoshm
rekhajoshm committed Jul 8, 2015
1 parent ab65fa1 commit a41ac4b
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 108 deletions.
7 changes: 4 additions & 3 deletions core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
/**
* Comparison function that defines the sort order for application attempts within the same
* application. Order is: running attempts before complete attempts, running attempts sorted
* by start time, completed attempts sorted by end time.
* by start time showing whichever started first,
* completed attempts sorted by end time showing whichever ended first.
*
* Normally applications should have a single running attempt; but failure to call sc.stop()
* may cause multiple running attempts to show up.
Expand All @@ -425,9 +426,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
a1: FsApplicationAttemptInfo,
a2: FsApplicationAttemptInfo): Boolean = {
if (a1.completed == a2.completed) {
if (a1.completed) a1.endTime >= a2.endTime else a1.startTime >= a2.startTime
if (a1.completed) a1.endTime <= a2.endTime else a1.startTime <= a2.startTime
} else {
!a1.completed
a1.completed
}
}

Expand Down
187 changes: 82 additions & 105 deletions core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
Option(request.getParameter("showIncomplete")).getOrElse("false").toBoolean

val allApps = parent.getApplicationList()
.filter(_.attempts.head.completed != requestedIncomplete)
val allAppsSize = allApps.size

val actualFirst = if (requestedFirst < allAppsSize) requestedFirst else 0
Expand All @@ -50,15 +51,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
val hasMultipleAttempts = appsToShow.exists(_.attempts.size > 1)
val appTable =
if (hasMultipleAttempts) {
UIUtils.listingTable(
appWithAttemptHeader,
appWithAttemptRow(_, requestedIncomplete),
appsToShow)
UIUtils.listingTable(appWithAttemptHeader, appWithAttemptRow, appsToShow)
} else {
UIUtils.listingTable(
appHeader,
appRow(_, requestedIncomplete),
appsToShow)
UIUtils.listingTable(appHeader, appRow, appsToShow)
}

val providerConfig = parent.getProviderConfig()
Expand All @@ -69,61 +64,61 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
{providerConfig.map { case (k, v) => <li><strong>{k}:</strong> {v}</li> }}
</ul>
{
// This displays the indices of pages that are within `plusOrMinus` pages of
// the current page. Regardless of where the current page is, this also links
// to the first and last page. If the current page +/- `plusOrMinus` is greater
// than the 2nd page from the first page or less than the 2nd page from the last
// page, `...` will be displayed.
if (allAppsSize > 0) {
val leftSideIndices =
rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete)
val rightSideIndices =
rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount,
requestedIncomplete)

<h4>
Showing {actualFirst + 1}-{last + 1} of {allAppsSize}
{if (requestedIncomplete) "(Incomplete applications)"}
<span style="float: right">
{
if (actualPage > 1) {
<a href={makePageLink(actualPage - 1, requestedIncomplete)}>&lt; </a>
<a href={makePageLink(1, requestedIncomplete)}>1</a>
}
}
{if (actualPage - plusOrMinus > secondPageFromLeft) " ... "}
{leftSideIndices}
{actualPage}
{rightSideIndices}
{if (actualPage + plusOrMinus < secondPageFromRight) " ... "}
{
if (actualPage < pageCount) {
<a href={makePageLink(pageCount, requestedIncomplete)}>{pageCount}</a>
<a href={makePageLink(actualPage + 1, requestedIncomplete)}> &gt;</a>
}
}
</span>
</h4> ++
// This displays the indices of pages that are within `plusOrMinus` pages of
// the current page. Regardless of where the current page is, this also links
// to the first and last page. If the current page +/- `plusOrMinus` is greater
// than the 2nd page from the first page or less than the 2nd page from the last
// page, `...` will be displayed.
if (allAppsSize > 0) {
val leftSideIndices =
rangeIndices(actualPage - plusOrMinus until actualPage, 1 < _, requestedIncomplete)
val rightSideIndices =
rangeIndices(actualPage + 1 to actualPage + plusOrMinus, _ < pageCount,
requestedIncomplete)

<h4>
Showing {actualFirst + 1}-{last + 1} of {allAppsSize}
{if (requestedIncomplete) "(Incomplete applications)"}
<span style="float: right">
{
if (actualPage > 1) {
<a href={makePageLink(actualPage - 1, requestedIncomplete)}>&lt; </a>
<a href={makePageLink(1, requestedIncomplete)}>1</a>
}
}
{if (actualPage - plusOrMinus > secondPageFromLeft) " ... "}
{leftSideIndices}
{actualPage}
{rightSideIndices}
{if (actualPage + plusOrMinus < secondPageFromRight) " ... "}
{
if (actualPage < pageCount) {
<a href={makePageLink(pageCount, requestedIncomplete)}>{pageCount}</a>
<a href={makePageLink(actualPage + 1, requestedIncomplete)}> &gt;</a>
}
}
</span>
</h4> ++
appTable
} else if (requestedIncomplete) {
<h4>No incomplete applications found!</h4>
} else {
<h4>No completed applications found!</h4> ++
} else if (requestedIncomplete) {
<h4>No incomplete applications found!</h4>
} else {
<h4>No completed applications found!</h4> ++
<p>Did you specify the correct logging directory?
Please verify your setting of <span style="font-style:italic">
spark.history.fs.logDirectory</span> and whether you have the permissions to
access it.<br /> It is also possible that your application did not run to
completion or did not stop the SparkContext.
</p>
}
}
}
<a href={makePageLink(actualPage, !requestedIncomplete)}>
{
if (requestedIncomplete) {
"Back to completed applications"
} else {
"Show incomplete applications"
}
if (requestedIncomplete) {
"Back to completed applications"
} else {
"Show incomplete applications"
}
}
</a>
</div>
Expand Down Expand Up @@ -151,19 +146,18 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
"Last Updated")

private def rangeIndices(
range: Seq[Int],
condition: Int => Boolean,
showIncomplete: Boolean): Seq[Node] = {
range: Seq[Int],
condition: Int => Boolean,
showIncomplete: Boolean): Seq[Node] = {
range.filter(condition).map(nextPage =>
<a href={makePageLink(nextPage, showIncomplete)}> {nextPage} </a>)
}

private def attemptRow(
renderAttemptIdColumn: Boolean,
info: ApplicationHistoryInfo,
attempt: ApplicationAttemptInfo,
isFirst: Boolean,
requestedIncomplete: Boolean): Seq[Node] = {
renderAttemptIdColumn: Boolean,
info: ApplicationHistoryInfo,
attempt: ApplicationAttemptInfo,
isFirst: Boolean): Seq[Node] = {
val uiAddress = HistoryServer.getAttemptURI(info.id, attempt.attemptId)
val startTime = UIUtils.formatDate(attempt.startTime)
val endTime = if (attempt.endTime > 0) UIUtils.formatDate(attempt.endTime) else "-"
Expand All @@ -174,67 +168,50 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
"-"
}
val lastUpdated = UIUtils.formatDate(attempt.lastUpdated)
var someAttemptCompleted = false
info.attempts.foreach{ attempt =>
if (attempt.completed) someAttemptCompleted = true
}
<tr>
{
if (isFirst) {
if (info.attempts.size > 1 || renderAttemptIdColumn) {
<td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
<a href={uiAddress}>{info.id}</a></td>
if (isFirst) {
if (info.attempts.size > 1 || renderAttemptIdColumn) {
<td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
<a href={uiAddress}>{info.id}</a></td>
<td rowspan={info.attempts.size.toString} style="background-color: #ffffff">
{info.name}</td>
} else {
<td><a href={uiAddress}>{info.id}</a></td>
<td>{info.name}</td>
}
} else {
Nil
<td><a href={uiAddress}>{info.id}</a></td>
<td>{info.name}</td>
}
} else {
Nil
}
{
if (renderAttemptIdColumn &&
(requestedIncomplete || (!requestedIncomplete && someAttemptCompleted))) {
if (info.attempts.size > 1 && attempt.attemptId.isDefined) {
<td><a href={HistoryServer.getAttemptURI(info.id, attempt.attemptId)}>
{attempt.attemptId.get}</a></td>
} else {
<td>&nbsp;</td>
}
} else {
Nil
}
}
{
if (requestedIncomplete || (!requestedIncomplete && someAttemptCompleted)) {
<td sorttable_customkey={attempt.startTime.toString}>{startTime}</td>
<td sorttable_customkey={attempt.endTime.toString}>{endTime}</td>
<td sorttable_customkey={(attempt.endTime - attempt.startTime).toString}>
{duration}</td>
<td>{attempt.sparkUser}</td>
<td sorttable_customkey={attempt.lastUpdated.toString}>{lastUpdated}</td>
if (renderAttemptIdColumn) {
if (info.attempts.size > 1 && attempt.attemptId.isDefined) {
<td><a href={HistoryServer.getAttemptURI(info.id, attempt.attemptId)}>
{attempt.attemptId.get}</a></td>
} else {
Nil
<td>&nbsp;</td>
}
} else {
Nil
}
}
<td sorttable_customkey={attempt.startTime.toString}>{startTime}</td>
<td sorttable_customkey={attempt.endTime.toString}>{endTime}</td>
<td sorttable_customkey={(attempt.endTime - attempt.startTime).toString}>
{duration}</td>
<td>{attempt.sparkUser}</td>
<td sorttable_customkey={attempt.lastUpdated.toString}>{lastUpdated}</td>
</tr>
}



private def appRow(
info: ApplicationHistoryInfo,
requestedIncomplete: Boolean): Seq[Node] = {
attemptRow(false, info, info.attempts.head, true, requestedIncomplete)
private def appRow(info: ApplicationHistoryInfo): Seq[Node] = {
attemptRow(false, info, info.attempts.head, true)
}

private def appWithAttemptRow(
info: ApplicationHistoryInfo,
requestedIncomplete: Boolean): Seq[Node] = {
attemptRow(true, info, info.attempts.head, true, requestedIncomplete) ++
info.attempts.drop(1).flatMap(attemptRow(true, info, _, false, requestedIncomplete))
private def appWithAttemptRow(info: ApplicationHistoryInfo): Seq[Node] = {
attemptRow(true, info, info.attempts.head, true) ++
info.attempts.drop(1).flatMap(attemptRow(true, info, _, false))
}

private def makePageLink(linkPage: Int, showIncomplete: Boolean): String = {
Expand Down

0 comments on commit a41ac4b

Please sign in to comment.