Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-20391][Core] Rename memory related fields in ExecutorSummay #17700

Closed
wants to merge 2 commits into from
Closed

[SPARK-20391][Core] Rename memory related fields in ExecutorSummay #17700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
135e233
Rename memory related fields in ExecutorSummay
jerryshao Apr 20, 2017
e1d618b
Address the comment to use MemoryMetrics instead
jerryshao Apr 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 26 additions & 22 deletions core/src/main/resources/org/apache/spark/ui/static/executorspage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,21 +253,25 @@ $(document).ready(function () {
var deadTotalBlacklisted = 0;

response.forEach(function (exec) {
exec.onHeapMemoryUsed = exec.hasOwnProperty('onHeapMemoryUsed') ? exec.onHeapMemoryUsed : 0;
exec.maxOnHeapMemory = exec.hasOwnProperty('maxOnHeapMemory') ? exec.maxOnHeapMemory : 0;
exec.offHeapMemoryUsed = exec.hasOwnProperty('offHeapMemoryUsed') ? exec.offHeapMemoryUsed : 0;
exec.maxOffHeapMemory = exec.hasOwnProperty('maxOffHeapMemory') ? exec.maxOffHeapMemory : 0;
var memoryMetrics = {
usedOnHeapStorageMemory: 0,
usedOffHeapStorageMemory: 0,
totalOnHeapStorageMemory: 0,
totalOffHeapStorageMemory: 0
};

exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
});

response.forEach(function (exec) {
allExecCnt += 1;
allRDDBlocks += exec.rddBlocks;
allMemoryUsed += exec.memoryUsed;
allMaxMemory += exec.maxMemory;
allOnHeapMemoryUsed += exec.onHeapMemoryUsed;
allOnHeapMaxMemory += exec.maxOnHeapMemory;
allOffHeapMemoryUsed += exec.offHeapMemoryUsed;
allOffHeapMaxMemory += exec.maxOffHeapMemory;
allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
allDiskUsed += exec.diskUsed;
allTotalCores += exec.totalCores;
allMaxTasks += exec.maxTasks;
Expand All @@ -286,10 +290,10 @@ $(document).ready(function () {
activeRDDBlocks += exec.rddBlocks;
activeMemoryUsed += exec.memoryUsed;
activeMaxMemory += exec.maxMemory;
activeOnHeapMemoryUsed += exec.onHeapMemoryUsed;
activeOnHeapMaxMemory += exec.maxOnHeapMemory;
activeOffHeapMemoryUsed += exec.offHeapMemoryUsed;
activeOffHeapMaxMemory += exec.maxOffHeapMemory;
activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
activeDiskUsed += exec.diskUsed;
activeTotalCores += exec.totalCores;
activeMaxTasks += exec.maxTasks;
Expand All @@ -308,10 +312,10 @@ $(document).ready(function () {
deadRDDBlocks += exec.rddBlocks;
deadMemoryUsed += exec.memoryUsed;
deadMaxMemory += exec.maxMemory;
deadOnHeapMemoryUsed += exec.onHeapMemoryUsed;
deadOnHeapMaxMemory += exec.maxOnHeapMemory;
deadOffHeapMemoryUsed += exec.offHeapMemoryUsed;
deadOffHeapMaxMemory += exec.maxOffHeapMemory;
deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
deadDiskUsed += exec.diskUsed;
deadTotalCores += exec.totalCores;
deadMaxTasks += exec.maxTasks;
Expand Down Expand Up @@ -431,10 +435,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.onHeapMemoryUsed;
return row.memoryMetrics.usedOnHeapStorageMemory;
else
return (formatBytes(row.onHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOnHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('on_heap_memory')
Expand All @@ -443,10 +447,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.offHeapMemoryUsed;
return row.memoryMetrics.usedOffHeapStorageMemory;
else
return (formatBytes(row.offHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOffHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('off_heap_memory')
Expand Down
11 changes: 7 additions & 4 deletions core/src/main/scala/org/apache/spark/status/api/v1/api.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,13 @@ class ExecutorSummary private[spark](
val isBlacklisted: Boolean,
val maxMemory: Long,
val executorLogs: Map[String, String],
val onHeapMemoryUsed: Option[Long],
val offHeapMemoryUsed: Option[Long],
val maxOnHeapMemory: Option[Long],
val maxOffHeapMemory: Option[Long])
val memoryMetrics: Option[MemoryMetrics])

class MemoryMetrics private[spark](
val usedOnHeapStorageMemory: Long,
val usedOffHeapStorageMemory: Long,
val totalOnHeapStorageMemory: Long,
val totalOffHeapStorageMemory: Long)

class JobData private[spark](
val jobId: Int,
Expand Down
21 changes: 12 additions & 9 deletions core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest

import scala.xml.Node

import org.apache.spark.status.api.v1.ExecutorSummary
import org.apache.spark.status.api.v1.{ExecutorSummary, MemoryMetrics}
import org.apache.spark.ui.{UIUtils, WebUIPage}

// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
Expand Down Expand Up @@ -114,10 +114,16 @@ private[spark] object ExecutorsPage {
val rddBlocks = status.numBlocks
val memUsed = status.memUsed
val maxMem = status.maxMem
val onHeapMemUsed = status.onHeapMemUsed
val offHeapMemUsed = status.offHeapMemUsed
val maxOnHeapMem = status.maxOnHeapMem
val maxOffHeapMem = status.maxOffHeapMem
val memoryMetrics = for {
onHeapUsed <- status.onHeapMemUsed
offHeapUsed <- status.offHeapMemUsed
maxOnHeap <- status.maxOnHeapMem
maxOffHeap <- status.maxOffHeapMem
} yield {
new MemoryMetrics(onHeapUsed, offHeapUsed, maxOnHeap, maxOffHeap)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just want to double check -- there won't be any issues with event logs from spark 2.1, and only some of these fields being present, right? taking a quick look at 2.1 and your last change #14617, this looks right.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is to make sure memoryMetrics field will only be existed when replaying new event logs, for Spark 2.1- event log, this field will not be present.

}


val diskUsed = status.diskUsed
val taskSummary = listener.executorToTaskSummary.getOrElse(execId, ExecutorTaskSummary(execId))

Expand All @@ -142,10 +148,7 @@ private[spark] object ExecutorsPage {
taskSummary.isBlacklisted,
maxMem,
taskSummary.executorLogs,
onHeapMemUsed,
offHeapMemUsed,
maxOnHeapMem,
maxOffHeapMem
memoryMetrics
)
}
}
51 changes: 30 additions & 21 deletions core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
Expand All @@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
Expand All @@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},

"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
Expand All @@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
Expand All @@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]
51 changes: 30 additions & 21 deletions .../src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
Expand All @@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
Expand All @@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},

"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
Expand All @@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
Expand All @@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]