Skip to content

Commit

Permalink
[SPARK-20391][CORE] Rename memory related fields in ExecutorSummay
Browse files Browse the repository at this point in the history 
## What changes were proposed in this pull request?

This is a follow-up of #14617 to make the name of memory related fields more meaningful.

Here  for the backward compatibility, I didn't change `maxMemory` and `memoryUsed` fields.

## How was this patch tested?

Existing UT and local verification.

CC squito and tgravescs .

Author: jerryshao <[email protected]>

Closes #17700 from jerryshao/SPARK-20391.

(cherry picked from commit 66dd5b8)
Signed-off-by: Imran Rashid <[email protected]>
  • Loading branch information
@jerryshao @squito
jerryshao authored and squito committed Apr 26, 2017
1 parent 34dec68 commit b65858b
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 77 deletions.
48 changes: 26 additions & 22 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,21 +253,25 @@ $(document).ready(function () {
var deadTotalBlacklisted = 0;

response.forEach(function (exec) {
exec.onHeapMemoryUsed = exec.hasOwnProperty('onHeapMemoryUsed') ? exec.onHeapMemoryUsed : 0;
exec.maxOnHeapMemory = exec.hasOwnProperty('maxOnHeapMemory') ? exec.maxOnHeapMemory : 0;
exec.offHeapMemoryUsed = exec.hasOwnProperty('offHeapMemoryUsed') ? exec.offHeapMemoryUsed : 0;
exec.maxOffHeapMemory = exec.hasOwnProperty('maxOffHeapMemory') ? exec.maxOffHeapMemory : 0;
var memoryMetrics = {
usedOnHeapStorageMemory: 0,
usedOffHeapStorageMemory: 0,
totalOnHeapStorageMemory: 0,
totalOffHeapStorageMemory: 0
};

exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
});

response.forEach(function (exec) {
allExecCnt += 1;
allRDDBlocks += exec.rddBlocks;
allMemoryUsed += exec.memoryUsed;
allMaxMemory += exec.maxMemory;
allOnHeapMemoryUsed += exec.onHeapMemoryUsed;
allOnHeapMaxMemory += exec.maxOnHeapMemory;
allOffHeapMemoryUsed += exec.offHeapMemoryUsed;
allOffHeapMaxMemory += exec.maxOffHeapMemory;
allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
allDiskUsed += exec.diskUsed;
allTotalCores += exec.totalCores;
allMaxTasks += exec.maxTasks;
Expand All @@ -286,10 +290,10 @@ $(document).ready(function () {
activeRDDBlocks += exec.rddBlocks;
activeMemoryUsed += exec.memoryUsed;
activeMaxMemory += exec.maxMemory;
activeOnHeapMemoryUsed += exec.onHeapMemoryUsed;
activeOnHeapMaxMemory += exec.maxOnHeapMemory;
activeOffHeapMemoryUsed += exec.offHeapMemoryUsed;
activeOffHeapMaxMemory += exec.maxOffHeapMemory;
activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
activeDiskUsed += exec.diskUsed;
activeTotalCores += exec.totalCores;
activeMaxTasks += exec.maxTasks;
Expand All @@ -308,10 +312,10 @@ $(document).ready(function () {
deadRDDBlocks += exec.rddBlocks;
deadMemoryUsed += exec.memoryUsed;
deadMaxMemory += exec.maxMemory;
deadOnHeapMemoryUsed += exec.onHeapMemoryUsed;
deadOnHeapMaxMemory += exec.maxOnHeapMemory;
deadOffHeapMemoryUsed += exec.offHeapMemoryUsed;
deadOffHeapMaxMemory += exec.maxOffHeapMemory;
deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
deadDiskUsed += exec.diskUsed;
deadTotalCores += exec.totalCores;
deadMaxTasks += exec.maxTasks;
Expand Down Expand Up @@ -431,10 +435,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.onHeapMemoryUsed;
return row.memoryMetrics.usedOnHeapStorageMemory;
else
return (formatBytes(row.onHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOnHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('on_heap_memory')
Expand All @@ -443,10 +447,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.offHeapMemoryUsed;
return row.memoryMetrics.usedOffHeapStorageMemory;
else
return (formatBytes(row.offHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOffHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('off_heap_memory')
Expand Down
11 changes: 7 additions & 4 deletions core/src/main/scala/org/apache/spark/status/api/v1/api.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,13 @@ class ExecutorSummary private[spark](
val isBlacklisted: Boolean,
val maxMemory: Long,
val executorLogs: Map[String, String],
val onHeapMemoryUsed: Option[Long],
val offHeapMemoryUsed: Option[Long],
val maxOnHeapMemory: Option[Long],
val maxOffHeapMemory: Option[Long])
val memoryMetrics: Option[MemoryMetrics])

class MemoryMetrics private[spark](
val usedOnHeapStorageMemory: Long,
val usedOffHeapStorageMemory: Long,
val totalOnHeapStorageMemory: Long,
val totalOffHeapStorageMemory: Long)

class JobData private[spark](
val jobId: Int,
Expand Down
21 changes: 12 additions & 9 deletions core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest

import scala.xml.Node

import org.apache.spark.status.api.v1.ExecutorSummary
import org.apache.spark.status.api.v1.{ExecutorSummary, MemoryMetrics}
import org.apache.spark.ui.{UIUtils, WebUIPage}

// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
Expand Down Expand Up @@ -114,10 +114,16 @@ private[spark] object ExecutorsPage {
val rddBlocks = status.numBlocks
val memUsed = status.memUsed
val maxMem = status.maxMem
val onHeapMemUsed = status.onHeapMemUsed
val offHeapMemUsed = status.offHeapMemUsed
val maxOnHeapMem = status.maxOnHeapMem
val maxOffHeapMem = status.maxOffHeapMem
val memoryMetrics = for {
onHeapUsed <- status.onHeapMemUsed
offHeapUsed <- status.offHeapMemUsed
maxOnHeap <- status.maxOnHeapMem
maxOffHeap <- status.maxOffHeapMem
} yield {
new MemoryMetrics(onHeapUsed, offHeapUsed, maxOnHeap, maxOffHeap)
}


val diskUsed = status.diskUsed
val taskSummary = listener.executorToTaskSummary.getOrElse(execId, ExecutorTaskSummary(execId))

Expand All @@ -142,10 +148,7 @@ private[spark] object ExecutorsPage {
taskSummary.isBlacklisted,
maxMem,
taskSummary.executorLogs,
onHeapMemUsed,
offHeapMemUsed,
maxOnHeapMem,
maxOffHeapMem
memoryMetrics
)
}
}
51 changes: 30 additions & 21 deletions core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
Expand All @@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
Expand All @@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},

"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
Expand All @@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
Expand All @@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]
51 changes: 30 additions & 21 deletions .../src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
Expand All @@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
Expand All @@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},

"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
Expand All @@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
Expand All @@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]

0 comments on commit b65858b

Please sign in to comment.