Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added GPU & Memory util meters with -m flag #25

Merged
merged 3 commits into from
Mar 23, 2024
Merged

Added GPU & Memory util meters with -m flag #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c50e318
Added GPU & Memory util meters with -m flag
rit628 Feb 11, 2024
1edbe18
Moved gpu util meter to rightmost column
rit628 Feb 11, 2024
ee7940c
Added unit tests for meter functionality
rit628 Mar 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions nvidia-htop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
parser = argparse.ArgumentParser()
parser.add_argument('-l', '--command-length', default=20, const=100, type=int, nargs='?')
parser.add_argument('-c', '--color', action='store_true')
parser.add_argument('-m', '--meter', action='store_true', help="Shows meters for GPU and VRAM utilization")
parser.add_argument('-u', '--user', default='', help="Limit the list of processes to selected users (comma-separated)")
parser.add_argument('-i', '--id', default='', help="Limit the command to selected GPU IDs (comma-separated)")
# only for testing
Expand All @@ -41,6 +42,7 @@
# parse the command length argument
command_length = args.command_length
color = args.color
meter = args.meter
fake_ps = args.fake_ps
users = set(args.user.split(',')) if len(args.user) > 0 else None

Expand Down Expand Up @@ -71,8 +73,31 @@
lines = [line + '\n' for line in lines_proc[:-1]]
lines += lines_proc[-1]

def add_meters(_lines):
pattern = re.compile(r"\| (?:N/A|..%)\s+[0-9]{2,3}C.*\s([0-9]+)MiB\s+/\s+([0-9]+)MiB.*\s([0-9]+)%")
gpu_util_lines = list(filter(lambda tup: pattern.match(tup[1]), enumerate(_lines)))
for i, line in gpu_util_lines:
m = pattern.match(line)
used_mem = int(m.group(1))
total_mem = int(m.group(2))
gpu_util = int(m.group(3)) / 100.0
mem_util = used_mem / float(total_mem)
# Uses empty space underneath gpu & mem util stats for meter placement.
meter_space = re.split(r"\|(?!$)", _lines[i+1])
gpu_util_space, mig, _ = re.split(r"([^ |]+ \|)", meter_space[-1], maxsplit=1)
gpu_meter_space = len(gpu_util_space)-4
mem_meter_space = len(meter_space[2])-4
# Fill gpu and mem util meters proportional to reported utilization
gpu_meter = "|"*round(gpu_util*gpu_meter_space) + " "*round((1-gpu_util)*gpu_meter_space)
mem_meter = "|"*round(mem_util*mem_meter_space) + " "*round((1-mem_util)*mem_meter_space)
meter_space[-1] = f" [{gpu_meter}] {mig}"
meter_space[2] = f" [{mem_meter}] "
_lines[i+1] = '|'.join(meter_space)

return _lines

def colorize(_lines):
_lines = add_meters(_lines) if meter else _lines
# Index of the first content line of the current cell in the nvidia-smi output.
start_idx = 0
for j in range(len(_lines)):
Expand Down Expand Up @@ -118,6 +143,9 @@ def colorize(_lines):
if color:
lines_to_print = colorize(lines_to_print)

elif meter:
lines_to_print = add_meters(lines_to_print)

# we print all but the last line which is the +---+ separator
for line in lines_to_print[:-1]:
print(line)
Expand Down
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-----------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_COLOR
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-----------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_L
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_L150
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 11021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB/matlab94/bin/glnxa64/MATLAB script.m |
| 1 14518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
26 changes: 26 additions & 0 deletions test/DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Sun Aug 2 13:44:21 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57 Driver Version: 450.57 CUDA Version: 11.0 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 GeForce RTX 2070 Off | 00000000:01:00.0 Off | N/A |
| 0% 50C P2 27W / 175W | 749MiB / 7981MiB | 2% Default |
| | [|| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+
| 1 GeForce GTX 750 Ti Off | 00000000:04:00.0 On | N/A |
| 33% 30C P8 1W / 46W | 871MiB / 2002MiB | 0% Default |
| | [|||||||| ] | [ ] N/A |
+-------------------------------+----------------------+----------------------+

+-------------------------------------------------------------------------------+
| GPU PID USER GPU MEM %CPU %MEM TIME COMMAND |
| 0 1032 root 745MiB 10.0 5.0 11:42:17 python 0.py |
| 1 1111021 admin 244MiB 1.0 4.0 12:42:17 python3 1.py |
| 1 25544 test 139MiB 5.0 3.0 13:42:17 python 2.py |
| 1 4755 user1 3MiB 3.0 2.0 14:42:17 /opt/software/MATLAB |
| 1 1114518 root 1MiB 8.0 1.0 15:42:17 python3 3.py |
| 1 13956 root 472MiB 2.0 0.0 16:42:17 python3 4.py |
+-------------------------------------------------------------------------------+
16 changes: 16 additions & 0 deletions test/test_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,22 @@ def test_no_processes(self):
def test_no_processes_docker(self):
self.do_test('FAKE_STDIN_NO_PROCESSES_DOCKER', 'DESIRED_STDOUT_NO_PROCESSES_DOCKER')

def test_with_meters(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER', call_args=["-m"])

def test_with_meters_color(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_COLOR', call_args=["-m", "-c"])

def test_with_meters_long_pids(self):
self.do_test('FAKE_STDIN_LONG_PIDS', 'DESIRED_STDOUT_NEW_FORMAT_METER_LONG_PIDS', call_args=["-m"], fake_ps='FAKE_PS_LONG_PIDS')

def test_with_meters_long(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L', call_args=["-m", "-l"])

def test_with_meters_very_long(self):
self.do_test('FAKE_STDIN_NEW_FORMAT', 'DESIRED_STDOUT_NEW_FORMAT_METER_L150', call_args=["-m", "-l", "150"])



if __name__ == '__main__':
unittest.main()
Loading