-
Notifications
You must be signed in to change notification settings - Fork 37
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix implementation of processSQLPlanMetrics in Profiler #853
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,7 +29,7 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
|
||
def getDurations(tcs: ArrayBuffer[TaskCase]): (Long, Long, Long, Double) = { | ||
val durations = tcs.map(_.duration) | ||
if (durations.size > 0 ) { | ||
if (durations.nonEmpty ) { | ||
(durations.sum, durations.max, durations.min, | ||
ToolUtils.calculateAverage(durations.sum, durations.size, 1)) | ||
} else { | ||
|
@@ -49,22 +49,20 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
def jobAndStageMetricsAggregation(): Seq[JobStageAggTaskMetricsProfileResult] = { | ||
val allJobRows = apps.flatMap { app => | ||
app.jobIdToInfo.map { case (id, jc) => | ||
val stageIdsInJob = jc.stageIds | ||
val stagesInJob = app.stageIdToInfo.filterKeys { case (sid, _) => | ||
stageIdsInJob.contains(sid) | ||
}.keys.map(_._1).toSeq | ||
jc.stageIds.contains(sid) | ||
}.keys.map(_._1).toSet | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use |
||
if (stagesInJob.isEmpty) { | ||
None | ||
} else { | ||
val tasksInJob = app.taskEnd.filter { tc => | ||
stagesInJob.contains(tc.stageId) | ||
} | ||
// count duplicate task attempts | ||
val numTaskAttempt = tasksInJob.size | ||
val (durSum, durMax, durMin, durAvg) = getDurations(tasksInJob) | ||
Some(JobStageAggTaskMetricsProfileResult(app.index, | ||
s"job_$id", | ||
numTaskAttempt, | ||
tasksInJob.size, | ||
nartal1 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
jc.duration, | ||
tasksInJob.map(_.diskBytesSpilled).sum, | ||
durSum, | ||
|
@@ -100,9 +98,8 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
} | ||
val allJobStageRows = apps.flatMap { app => | ||
app.jobIdToInfo.flatMap { case (_, jc) => | ||
val stageIdsInJob = jc.stageIds | ||
val stagesInJob = app.stageIdToInfo.filterKeys { case (sid, _) => | ||
stageIdsInJob.contains(sid) | ||
jc.stageIds.contains(sid) | ||
} | ||
if (stagesInJob.isEmpty) { | ||
None | ||
|
@@ -153,17 +150,16 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
} | ||
// stages that are missing from a job, perhaps dropped events | ||
val stagesWithoutJobs = apps.flatMap { app => | ||
val allStageinJobs = app.jobIdToInfo.flatMap { case (_, jc) => | ||
val stageIdsInJob = jc.stageIds | ||
val allStageInJobs = app.jobIdToInfo.flatMap { case (_, jc) => | ||
app.stageIdToInfo.filterKeys { case (sid, _) => | ||
stageIdsInJob.contains(sid) | ||
jc.stageIds.contains(sid) | ||
} | ||
} | ||
val missing = app.stageIdToInfo.keys.toSeq.diff(allStageinJobs.keys.toSeq) | ||
val missing = app.stageIdToInfo.keys.toSet.diff(allStageInJobs.keys.toSet) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sets are more time efficient compared to sequences |
||
if (missing.isEmpty) { | ||
Seq.empty | ||
} else { | ||
missing.map { case ((id, saId)) => | ||
missing.map { case (id, saId) => | ||
val scOpt = app.stageIdToInfo.get((id, saId)) | ||
scOpt match { | ||
case None => | ||
|
@@ -215,10 +211,10 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
|
||
val allRows = allJobRows ++ allJobStageRows ++ stagesWithoutJobs | ||
val filteredRows = allRows.filter(_.isDefined).map(_.get) | ||
nartal1 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (filteredRows.size > 0) { | ||
if (filteredRows.nonEmpty) { | ||
val sortedRows = filteredRows.sortBy { cols => | ||
val sortDur = cols.duration.getOrElse(0L) | ||
(cols.appIndex, -(sortDur), cols.id) | ||
(cols.appIndex, -sortDur, cols.id) | ||
} | ||
sortedRows | ||
} else { | ||
|
@@ -231,12 +227,12 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
val allRows = apps.flatMap { app => | ||
app.sqlIdToInfo.map { case (sqlId, sqlCase) => | ||
val jcs = app.jobIdToInfo.filter { case (_, jc) => | ||
jc.sqlID.getOrElse(-1) == sqlId | ||
jc.sqlID.isDefined && jc.sqlID.get == sqlId | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
} | ||
if (jcs.isEmpty) { | ||
None | ||
} else { | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSeq | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSet | ||
val tasksInSQL = app.taskEnd.filter { tc => | ||
stageIdsForSQL.contains(tc.stageId) | ||
} | ||
|
@@ -314,12 +310,12 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
val allRows = apps.flatMap { app => | ||
app.sqlIdToInfo.map { case (sqlId, _) => | ||
val jcs = app.jobIdToInfo.filter { case (_, jc) => | ||
jc.sqlID.getOrElse(-1) == sqlId | ||
jc.sqlID.isDefined && jc.sqlID.get == sqlId | ||
} | ||
if (jcs.isEmpty) { | ||
None | ||
} else { | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSeq | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSet | ||
|
||
val tasksInSQL = app.taskEnd.filter { tc => | ||
stageIdsForSQL.contains(tc.stageId) | ||
|
@@ -359,12 +355,12 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
apps.map { app => | ||
val maxOfSqls = app.sqlIdToInfo.map { case (sqlId, _) => | ||
val jcs = app.jobIdToInfo.filter { case (_, jc) => | ||
jc.sqlID.getOrElse(-1) == sqlId | ||
jc.sqlID.isDefined && jc.sqlID.get == sqlId | ||
} | ||
if (jcs.isEmpty) { | ||
0L | ||
} else { | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSeq | ||
val stageIdsForSQL = jcs.flatMap(_._2.stageIds).toSet | ||
val tasksInSQL = app.taskEnd.filter { tc => | ||
stageIdsForSQL.contains(tc.stageId) | ||
} | ||
|
@@ -394,7 +390,7 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
sqlCase.sqlCpuTimePercent) | ||
} | ||
} | ||
if (allRows.size > 0) { | ||
if (allRows.nonEmpty) { | ||
val sortedRows = allRows.sortBy { cols => | ||
val sortDur = cols.duration.getOrElse(0L) | ||
(cols.appIndex, cols.sqlID, sortDur) | ||
|
@@ -444,7 +440,7 @@ class Analysis(apps: Seq[ApplicationInfo]) { | |
} | ||
|
||
val allNonEmptyRows = allRows.filter(_.isDefined).map(_.get) | ||
nartal1 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (allNonEmptyRows.size > 0) { | ||
if (allNonEmptyRows.nonEmpty) { | ||
val sortedRows = allNonEmptyRows.sortBy { cols => | ||
(cols.appIndex, cols.stageId, cols.stageAttemptId, cols.taskId, cols.taskAttemptId) | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
val stageIdsInJob = jc.stageIds
was removed to save memory compared to directly accessingjc.stageIds