Skip to content

Commit

Permalink
Merge pull request #606 from StanfordVL/profiling
Browse files Browse the repository at this point in the history 
Update Profiling
  • Loading branch information
@wensi-ai
wensi-ai authored Feb 15, 2024
2 parents e024b05 + 37169a0 commit 16bcc5f
Show file tree
Hide file tree
Showing 5 changed files with 26 additions and 25 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/profiling.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,5 @@ jobs:
fail-on-alert: false
alert-threshold: '200%'
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
comment-on-alert: false
auto-push: true
8 changes: 4 additions & 4 deletions scripts/profiling.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<link rel="stylesheet" href="profiling.css">
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.2/dist/Chart.min.js"></script>
</head>
<body>
<header id="header">
Expand All @@ -32,7 +32,7 @@ <h5>*Ran with GPU dynamics on</h5>
<div class="col-3">
<div class="list-group list-group-light" id="baseline_tab" role="tablist">
<a class="list-group-item list-group-item-action active px-2 border-0" id="baseline_total_tab"
data-toggle="list" href="#baseline_total" role="tab" aria-controls="baseline_total">Total Frame Time</a>
data-toggle="list" href="#baseline_total" role="tab" aria-controls="baseline_total">FPS</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="baseline_loading_tab"
data-toggle="list" href="#baseline_loading" role="tab" aria-controls="baseline_loading">Loading Time</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="baseline_omni_tab"
Expand Down Expand Up @@ -76,7 +76,7 @@ <h5>* Ran with GPU dynamics off</h5>
<div class="col-3">
<div class="list-group list-group-light" id="scene_tab" role="tablist">
<a class="list-group-item list-group-item-action active px-2 border-0" id="scene_total_tab"
data-toggle="list" href="#scene_total" role="tab" aria-controls="scene_total">Total Frame Time</a>
data-toggle="list" href="#scene_total" role="tab" aria-controls="scene_total">FPS</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="scene_loading_tab"
data-toggle="list" href="#scene_loading" role="tab" aria-controls="scene_loading">Loading Time</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="scene_omni_tab"
Expand Down Expand Up @@ -120,7 +120,7 @@ <h5>* Ran with GPU dynamics on</h5>
<div class="col-3">
<div class="list-group list-group-light" id="np_tab" role="tablist">
<a class="list-group-item list-group-item-action active px-2 border-0" id="np_total_tab"
data-toggle="list" href="#np_total" role="tab" aria-controls="np_total">Total Frame Time</a>
data-toggle="list" href="#np_total" role="tab" aria-controls="np_total">FPS</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="np_loading_tab"
data-toggle="list" href="#np_loading" role="tab" aria-controls="np_loading">Loading Time</a>
<a class="list-group-item list-group-item-action px-2 border-0" id="np_omni_tab"
Expand Down
20 changes: 10 additions & 10 deletions scripts/profiling.js
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
const canvasDict = {
'baseline_total_canvas': ["Total frame time", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_total_canvas': ["FPS", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_loading_canvas': ["Loading time", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_omni_canvas': ["Omni step time", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_non_omni_canvas': ["Non-omni step time", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_mem_canvas': ["Memory usage", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'baseline_vram_canvas': ["Vram usage", ["Empty scene", "Rs_int", "Rs_int, with 1 Fetch", "Rs_int, with 3 Fetch"]],
'np_total_canvas': ["Total frame time", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_total_canvas': ["FPS", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_loading_canvas': ["Loading time", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_omni_canvas': ["Omni step time", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_non_omni_canvas': ["Non-omni step time", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_mem_canvas': ["Memory usage", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'np_vram_canvas': ["Vram usage", ["Empty scene, with 1 Fetch, fluids", "Empty scene, with 1 Fetch, cloth", "Empty scene, with 1 Fetch, macro particles", "Empty scene, with 1 Fetch, cloth, fluids, macro particles"]],
'scene_total_canvas': ["Total frame time", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_loading_canvas': ["Loading time", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_omni_canvas': ["Omni step time", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_og_canvas': ["Non-omni step time", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_mem_canvas': ["Memory usage", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_vram_canvas': ["Vram usage", ["Rs_int, with 1 Fetch robot", "house_single_floor, with 1 Fetch robot", "grocery_store_cafe, with 1 Fetch robot", "Pomaria_0_garden, with 1 Fetch robot"]],
'scene_total_canvas': ["FPS", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
'scene_loading_canvas': ["Loading time", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
'scene_omni_canvas': ["Omni step time", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
'scene_non_omni_canvas': ["Non-omni step time", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
'scene_mem_canvas': ["Memory usage", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
'scene_vram_canvas': ["Vram usage", ["Ihlen_0_int, with 1 Fetch", "Pomaria_0_garden, with 1 Fetch", "house_single_floor, with 1 Fetch", "grocery_store_cafe, with 1 Fetch"]],
}


Expand Down Expand Up @@ -118,7 +118,7 @@ function renderGraph(canvasName, fieldName, runNames) {
},
afterLabel: item => {
const { extra } = filteredData.values().next().value[item.index].bench;
return extra ? '\n' + extra : '';
return extra ? '\n' + extra[0] : '';
}
}
},
Expand All @@ -127,7 +127,7 @@ function renderGraph(canvasName, fieldName, runNames) {
return;
}
// XXX: Undocumented. How can we know the index?
const index = activeElems[0].index;
const index = activeElems[0]._index;
const url = filteredData.values().next().value[index].commit.url;
window.open(url, '_blank');
},
Expand Down
4 changes: 2 additions & 2 deletions scripts/profiling.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# warm up isaac sim
python tests/benchmark/profiling.py -s Rs_int
python tests/benchmark/profiling.py -s Rs_int -g
rm output.json
# 1st batch: baselines
python tests/benchmark/profiling.py -g # baseline
Expand All @@ -8,7 +8,7 @@ python tests/benchmark/profiling.py -g -s Rs_int -r 1 # for robotics resea
python tests/benchmark/profiling.py -g -s Rs_int -r 3 # for multi-agent research

# 2nd batch: compare different scenes
python tests/benchmark/profiling.py -r 1 -s Rs_int
python tests/benchmark/profiling.py -r 1 -s Ihlen_0_int
python tests/benchmark/profiling.py -r 1 -s Pomaria_0_garden
python tests/benchmark/profiling.py -r 1 -s house_single_floor
python tests/benchmark/profiling.py -r 1 -s grocery_store_cafe
Expand Down
17 changes: 9 additions & 8 deletions tests/benchmark/profiling.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
parser.add_argument("-g", "--gpu_denamics", action='store_true')
parser.add_argument("-p", "--macro_particle_system", action='store_true')

PROFILING_FIELDS = ["Total frame time", "Omni step time", "Non-omni step time", "Memory usage", "Vram usage"]
PROFILING_FIELDS = ["FPS", "Omni step time", "Non-omni step time", "Memory usage", "Vram usage"]
NUM_CLOTH = 5
NUM_SLICE_OBJECT = 3

Expand All @@ -31,6 +31,7 @@
"Pomaria_0_garden": [0.3, 0],
"grocery_store_cafe": [-3.5, 3.5],
"house_single_floor": [-3, -1],
"Ihlen_0_int": [-1, 2]
}


Expand All @@ -40,7 +41,7 @@ def main():
gm.ENABLE_HQ_RENDERING = args.fluids
gm.ENABLE_OBJECT_STATES = True
gm.ENABLE_TRANSITION_RULES = True
gm.ENABLE_FLATCACHE = True
gm.ENABLE_FLATCACHE = not args.cloth
gm.USE_GPU_DYNAMICS = args.gpu_denamics

cfg = {
Expand Down Expand Up @@ -160,12 +161,12 @@ def main():
})
results = np.array(results)
for i, title in enumerate(PROFILING_FIELDS):
output.append({
"name": field,
"unit": "time (ms)" if 'time' in title else "GB",
"value": np.mean(results[:, i]),
"extra": [title, title]
})
unit = "time (ms)" if 'time' in title else "GB"
value = np.mean(results[:, i])
if title == "FPS":
value = 1000 / value
unit = "fps"
output.append({"name": field, "unit": unit, "value": value, "extra": [title, title]})

ret = []
if os.path.exists("output.json"):
Expand Down

0 comments on commit 16bcc5f

Please sign in to comment.