From d0f36ee4f55a1c7eaa28d4e46098d673d73eddd8 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 26 Sep 2019 16:37:18 -0700 Subject: [PATCH 01/21] adding draft for daemon getting process data --- .../procdockerstats/procdockerstats | 155 ++++++++++++++++++ .../procdockerstats/procdockerstats.service | 11 ++ 2 files changed, 166 insertions(+) create mode 100644 files/image_config/procdockerstats/procdockerstats create mode 100644 files/image_config/procdockerstats/procdockerstats.service diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats new file mode 100644 index 000000000000..6bad999f3e6c --- /dev/null +++ b/files/image_config/procdockerstats/procdockerstats @@ -0,0 +1,155 @@ +# !/usr/bin/env python +''' +This code is for a specific daemon, process-docker. +Which sends process and docker CPU/memory utilization data to DB every 2 mins. +''' + +import sys, errno +import os +import time +import syslog +import signal +import select +import subprocess +import re +import swsssdk +import collections +from datetime import datetime + +VERSION = '1.0' + +SYSLOG_IDENTIFIER = "process-docker" + +REDIS_HOSTIP = "127.0.0.1" + +# ========================== Syslog wrappers ========================== +def log_info(msg, also_print_to_console=False): + syslog.openlog(SYSLOG_IDENTIFIER) + syslog.syslog(syslog.LOG_INFO, msg) + syslog.closelog() + +def log_warning(msg, also_print_to_console=False): + syslog.openlog(SYSLOG_IDENTIFIER) + syslog.syslog(syslog.LOG_WARNING, msg) + syslog.closelog() + +def log_error(msg, also_print_to_console=False): + syslog.openlog(SYSLOG_IDENTIFIER) + syslog.syslog(syslog.LOG_ERR, msg) + syslog.closelog() + +# ========================== ProcessDocker class ========================== +class ProcDockerStats: + + def __init__(self): + self.swid = 0 + self.running = False + self.handle = None + + def initialize(self): + self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) + self.state_db.connect("STATE_DB") + # wipe out all data before updating with new values + self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') + + def run_commands(self, commands): + """ + Given a list of shell commands, run them in order + Args: + commands: List of strings, each string is a shell command + """ + for cmd in commands: + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + + if proc.returncode != 0: + log_error("Error running command '{}'".format(cmd)) + else: + lines = re.split("\n", stdout) + keys = re.split(" +", lines[0]) + x = len(lines) + dict1 = dict() + dict_list = [] + for i in range(1,x): + values1 = re.split(" +", lines[i]) + dict1 = dict(zip(keys, values1)) + dict_list.append(dict1) + return dict_list + + def update_process_docker_commands(self): + """ + Convenience wrapper which retrieves process and docker CPU/memory utilization data + """ + process_docker_cmds = [] + process_docker_cmds.append("docker stats --no-stream -a") + """ process_docker_cmds.append("ps aux")""" + + log_info("Issuing the following commands:") + for cmd in process_docker_cmds: + log_info(" " + cmd) + + dockerdata = self.run_commands(process_docker_cmds) + value = "" + + for index in range(len(dockerdata)): + row = dockerdata[index] + cid = row.get('CONTAINER ID') + if cid != "": + value = 'Docker_Stats|' + str(cid) + name = row.get('NAME') + self.update_state_db(value, 'NAME', name) + + cpu = row.get('CPU %') + self.update_state_db(value, 'CPU%', str(cpu)) + + splitcol = row.get('MEM USAGE / LIMIT') + memuse = re.split(" / ", str(splitcol)) + self.update_state_db(value, 'MEM', str(memuse[0])) + self.update_state_db(value, 'MEM_LIMIT', str(memuse[1])) + + mem = row.get('MEM %') + self.update_state_db(value, 'MEM %', str(mem)) + + splitcol = row.get('NET I/O') + netio = re.split(" / ", str(splitcol)) + self.update_state_db(value, 'NET_IN', str(netio[0])) + self.update_state_db(value, 'NET_OUT', str(netio[1])) + + splitcol = row.get('BLOCK I/O') + blocio = re.split(" / ", str(splitcol)) + self.update_state_db(value, 'BLOCK_IN', str(blocio[0])) + self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) + + pids = row.get('PIDS') + self.update_state_db(value, 'PIDS', pids) + + def update_state_db(self, key1, key2, value2): + self.state_db.set('STATE_DB', key1, key2, value2) + + def run(self): + self.update_process_docker_commands() + datetimeobj = datetime.now() + # Adding key to store latest update time. + self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj) + + + +# main start +def main(): + log_info("process-docker stats aemon starting up..") + + if not os.getuid() == 0: + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" + sys.exit(1) + + pd = ProcDockerStats() + pd.initialize() + # Data need to be updated every 2 mins. hence adding delay of 120 seconds + #time.sleep(120) + pd.run() + log_info("process-docker stats daemon exited") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/files/image_config/procdockerstats/procdockerstats.service b/files/image_config/procdockerstats/procdockerstats.service new file mode 100644 index 000000000000..514c484c06ba --- /dev/null +++ b/files/image_config/procdockerstats/procdockerstats.service @@ -0,0 +1,11 @@ +[Unit] +Description=Control Plane ACL configuration daemon +Requires=updategraph.service +After=updategraph.service + +[Service] +Type=simple +ExecStart=/usr/bin/procdockerstats + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 58b59dc65d949d048615598908194d96a5ace94b Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Mon, 30 Sep 2019 12:59:49 -0700 Subject: [PATCH 02/21] resolving PR comments, adding separate method for docker --- .../procdockerstats/procdockerstats | 115 +++++++----------- 1 file changed, 41 insertions(+), 74 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 6bad999f3e6c..ba7962eafceb 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -4,21 +4,18 @@ This code is for a specific daemon, process-docker. Which sends process and docker CPU/memory utilization data to DB every 2 mins. ''' -import sys, errno +import sys import os import time import syslog -import signal -import select import subprocess import re import swsssdk -import collections from datetime import datetime VERSION = '1.0' -SYSLOG_IDENTIFIER = "process-docker" +SYSLOG_IDENTIFIER = "procdockerstats" REDIS_HOSTIP = "127.0.0.1" @@ -42,114 +39,84 @@ def log_error(msg, also_print_to_console=False): class ProcDockerStats: def __init__(self): - self.swid = 0 - self.running = False - self.handle = None - - def initialize(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) - self.state_db.connect("STATE_DB") - # wipe out all data before updating with new values - self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') - - def run_commands(self, commands): - """ - Given a list of shell commands, run them in order - Args: - commands: List of strings, each string is a shell command - """ - for cmd in commands: - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - (stdout, stderr) = proc.communicate() + self.state_db.connect("STATE_DB") + + def run_command(self, cmd): + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + + if proc.returncode != 0: + log_error("Error running command '{}'".format(cmd)) + else: + return stdout + + def format_docker_cmd_output(self, cmdout): + lines = re.split("\n", cmdout) + keys = re.split(" +", lines[0]) + x = len(lines) + dict1 = dict() + dict_list = [] + for i in range(1,x): + values1 = re.split(" +", lines[i]) + dict1 = dict(zip(keys, values1)) + dict_list.append(dict1) + return dict_list + + def update_dockerstats_command(self): + data = self.run_command("docker stats --no-stream -a") + dockerdata = self.format_docker_cmd_output(data) - if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) - else: - lines = re.split("\n", stdout) - keys = re.split(" +", lines[0]) - x = len(lines) - dict1 = dict() - dict_list = [] - for i in range(1,x): - values1 = re.split(" +", lines[i]) - dict1 = dict(zip(keys, values1)) - dict_list.append(dict1) - return dict_list - - def update_process_docker_commands(self): - """ - Convenience wrapper which retrieves process and docker CPU/memory utilization data - """ - process_docker_cmds = [] - process_docker_cmds.append("docker stats --no-stream -a") - """ process_docker_cmds.append("ps aux")""" - - log_info("Issuing the following commands:") - for cmd in process_docker_cmds: - log_info(" " + cmd) - - dockerdata = self.run_commands(process_docker_cmds) value = "" - + # wipe out all data before updating with new values + self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') for index in range(len(dockerdata)): - row = dockerdata[index] + row = dockerdata[index] cid = row.get('CONTAINER ID') - if cid != "": + if cid: value = 'Docker_Stats|' + str(cid) name = row.get('NAME') self.update_state_db(value, 'NAME', name) - cpu = row.get('CPU %') self.update_state_db(value, 'CPU%', str(cpu)) - splitcol = row.get('MEM USAGE / LIMIT') memuse = re.split(" / ", str(splitcol)) self.update_state_db(value, 'MEM', str(memuse[0])) self.update_state_db(value, 'MEM_LIMIT', str(memuse[1])) - mem = row.get('MEM %') self.update_state_db(value, 'MEM %', str(mem)) - splitcol = row.get('NET I/O') netio = re.split(" / ", str(splitcol)) self.update_state_db(value, 'NET_IN', str(netio[0])) self.update_state_db(value, 'NET_OUT', str(netio[1])) - splitcol = row.get('BLOCK I/O') blocio = re.split(" / ", str(splitcol)) self.update_state_db(value, 'BLOCK_IN', str(blocio[0])) self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) - pids = row.get('PIDS') - self.update_state_db(value, 'PIDS', pids) - + self.update_state_db(value, 'PIDS', pids) + def update_state_db(self, key1, key2, value2): self.state_db.set('STATE_DB', key1, key2, value2) def run(self): - self.update_process_docker_commands() + self.update_dockerstats_command() datetimeobj = datetime.now() - # Adding key to store latest update time. + #Adding key to store latest update time. self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj) - - # main start def main(): log_info("process-docker stats aemon starting up..") - if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" - sys.exit(1) - + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" + sys.exit(1) pd = ProcDockerStats() - pd.initialize() # Data need to be updated every 2 mins. hence adding delay of 120 seconds - #time.sleep(120) + time.sleep(120) pd.run() log_info("process-docker stats daemon exited") - - + if __name__ == '__main__': main() \ No newline at end of file From d5cf0f04727d748ec56f434f9afa07dcacbcf6e1 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Tue, 1 Oct 2019 13:19:13 -0700 Subject: [PATCH 03/21] adjusting whitespace --- .../procdockerstats/procdockerstats | 42 ++++++++++--------- .../procdockerstats/procdockerstats.service | 3 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index ba7962eafceb..af698f2c8929 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -10,7 +10,7 @@ import time import syslog import subprocess import re -import swsssdk +import swsssdk from datetime import datetime VERSION = '1.0' @@ -19,36 +19,40 @@ SYSLOG_IDENTIFIER = "procdockerstats" REDIS_HOSTIP = "127.0.0.1" + # ========================== Syslog wrappers ========================== def log_info(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_INFO, msg) syslog.closelog() + def log_warning(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_WARNING, msg) syslog.closelog() + def log_error(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_ERR, msg) syslog.closelog() + # ========================== ProcessDocker class ========================== class ProcDockerStats: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) - self.state_db.connect("STATE_DB") - + self.state_db.connect("STATE_DB") + def run_command(self, cmd): proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - (stdout, stderr) = proc.communicate() - - if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) - else: + (stdout, stderr) = proc.communicate() + + if proc.returncode != 0: + log_error("Error running command '{}'".format(cmd)) + else: return stdout def format_docker_cmd_output(self, cmdout): @@ -57,13 +61,13 @@ class ProcDockerStats: x = len(lines) dict1 = dict() dict_list = [] - for i in range(1,x): + for i in range(1, x): values1 = re.split(" +", lines[i]) - dict1 = dict(zip(keys, values1)) + dict1 = dict(zip(keys, values1)) dict_list.append(dict1) return dict_list - - def update_dockerstats_command(self): + + def update_dockerstats_command(self): data = self.run_command("docker stats --no-stream -a") dockerdata = self.format_docker_cmd_output(data) @@ -94,29 +98,29 @@ class ProcDockerStats: self.update_state_db(value, 'BLOCK_IN', str(blocio[0])) self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) pids = row.get('PIDS') - self.update_state_db(value, 'PIDS', pids) - + self.update_state_db(value, 'PIDS', pids) + def update_state_db(self, key1, key2, value2): self.state_db.set('STATE_DB', key1, key2, value2) def run(self): self.update_dockerstats_command() datetimeobj = datetime.now() - #Adding key to store latest update time. + # Adding key to store latest update time. self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj) # main start def main(): log_info("process-docker stats aemon starting up..") if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" sys.exit(1) pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds time.sleep(120) pd.run() log_info("process-docker stats daemon exited") - + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/files/image_config/procdockerstats/procdockerstats.service b/files/image_config/procdockerstats/procdockerstats.service index 514c484c06ba..31169dd61058 100644 --- a/files/image_config/procdockerstats/procdockerstats.service +++ b/files/image_config/procdockerstats/procdockerstats.service @@ -8,4 +8,5 @@ Type=simple ExecStart=/usr/bin/procdockerstats [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target + From b085b5b8adf4a6991f327632fe20b0a6e4e1505b Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Tue, 1 Oct 2019 18:33:52 -0700 Subject: [PATCH 04/21] adding process data output parsing to daemon --- .../procdockerstats/procdockerstats | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index af698f2c8929..4496d859c5e4 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -10,72 +10,78 @@ import time import syslog import subprocess import re -import swsssdk +import swsssdk from datetime import datetime VERSION = '1.0' - SYSLOG_IDENTIFIER = "procdockerstats" - REDIS_HOSTIP = "127.0.0.1" - # ========================== Syslog wrappers ========================== def log_info(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_INFO, msg) syslog.closelog() - def log_warning(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_WARNING, msg) syslog.closelog() - def log_error(msg, also_print_to_console=False): syslog.openlog(SYSLOG_IDENTIFIER) syslog.syslog(syslog.LOG_ERR, msg) syslog.closelog() - # ========================== ProcessDocker class ========================== class ProcDockerStats: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) - self.state_db.connect("STATE_DB") - + self.state_db.connect("STATE_DB") + def run_command(self, cmd): proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) (stdout, stderr) = proc.communicate() - if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) + log_error("Error running command '{}'".format(cmd)) else: return stdout def format_docker_cmd_output(self, cmdout): lines = re.split("\n", cmdout) keys = re.split(" +", lines[0]) - x = len(lines) dict1 = dict() dict_list = [] - for i in range(1, x): - values1 = re.split(" +", lines[i]) - dict1 = dict(zip(keys, values1)) + for item in lines[1:]: + values1 = re.split(" +", item) + dict1 = dict(zip(keys, values1)) dict_list.append(dict1) return dict_list - def update_dockerstats_command(self): + def format_process_cmd_output(self, cmdout): + lines = re.split("\n", cmdout) + keys = re.split(" +", lines[0]) + keylist = list(filter(None, keys)) + dict1 = dict() + dict_list = [] + for item in lines[1:]: + values1 = re.split(" +", str(item)) + # To remove extra space before UID + val = list(filter(None, values1)) + # Merging extra columns created due to space in cmd ouput + val[8:] = [''.join(val[8:])] + dict1 = dict(zip(keylist, val)) + dict_list.append(dict1) + return dict_list + + def update_dockerstats_command(self): data = self.run_command("docker stats --no-stream -a") dockerdata = self.format_docker_cmd_output(data) - value = "" # wipe out all data before updating with new values self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') - for index in range(len(dockerdata)): - row = dockerdata[index] + for row in dockerdata[0:]: cid = row.get('CONTAINER ID') if cid: value = 'Docker_Stats|' + str(cid) @@ -99,7 +105,35 @@ class ProcDockerStats: self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) pids = row.get('PIDS') self.update_state_db(value, 'PIDS', pids) - + + def update_processstats_command(self): + + data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -1024") + processdata = self.format_process_cmd_output(data) + value = "" + # wipe out all data before updating with new values + self.state_db.delete_all_by_pattern('STATE_DB', 'Process_Stats|*') + for row in processdata[0:]: + cid = row.get('PID') + if cid: + value = 'Process_Stats|' + str(cid) + uid = row.get('UID') + self.update_state_db(value, 'UID', uid) + ppid = row.get('PPID') + self.update_state_db(value, 'PPID', ppid) + cpu = row.get('%CPU') + self.update_state_db(value, '%CPU', str(cpu)) + mem = row.get('%MEM') + self.update_state_db(value, '%MEM', str(mem)) + stime = row.get('STIME') + self.update_state_db(value, 'STIME', stime) + tty = row.get('TT') + self.update_state_db(value, 'TT', tty) + time = row.get('TIME') + self.update_state_db(value, 'TIME', time) + cmd = row.get('CMD') + self.update_state_db(value, 'CMD', cmd) + def update_state_db(self, key1, key2, value2): self.state_db.set('STATE_DB', key1, key2, value2) @@ -108,19 +142,21 @@ class ProcDockerStats: datetimeobj = datetime.now() # Adding key to store latest update time. self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj) + self.update_processstats_command() + self.update_state_db('Process_Stats|LastUpdateTime', 'lastupdate', datetimeobj) # main start def main(): log_info("process-docker stats aemon starting up..") if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" sys.exit(1) pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds time.sleep(120) pd.run() log_info("process-docker stats daemon exited") - + if __name__ == '__main__': main() From 9624514ad8fd4734c1e485b85e7f4535d52e1e98 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 2 Oct 2019 14:41:55 -0700 Subject: [PATCH 05/21] adjusting space --- files/image_config/procdockerstats/procdockerstats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 4496d859c5e4..d0de057d1533 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -55,7 +55,7 @@ class ProcDockerStats: dict_list = [] for item in lines[1:]: values1 = re.split(" +", item) - dict1 = dict(zip(keys, values1)) + dict1 = dict(zip(keys, values1)) dict_list.append(dict1) return dict_list @@ -75,7 +75,7 @@ class ProcDockerStats: dict_list.append(dict1) return dict_list - def update_dockerstats_command(self): + def update_dockerstats_command(self): data = self.run_command("docker stats --no-stream -a") dockerdata = self.format_docker_cmd_output(data) value = "" @@ -149,9 +149,9 @@ class ProcDockerStats: def main(): log_info("process-docker stats aemon starting up..") if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" - sys.exit(1) + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" + sys.exit(1) pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds time.sleep(120) From f1b2d8aa1d91f6078305158c6200e757e0ffc8f3 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Mon, 7 Oct 2019 12:08:47 -0700 Subject: [PATCH 06/21] removing trailing spaces --- .../procdockerstats/procdockerstats | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index d0de057d1533..d238317aba30 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -10,7 +10,7 @@ import time import syslog import subprocess import re -import swsssdk +import swsssdk from datetime import datetime VERSION = '1.0' @@ -38,13 +38,13 @@ class ProcDockerStats: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) - self.state_db.connect("STATE_DB") - + self.state_db.connect("STATE_DB") + def run_command(self, cmd): proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) (stdout, stderr) = proc.communicate() if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) + log_error("Error running command '{}'".format(cmd)) else: return stdout @@ -70,11 +70,11 @@ class ProcDockerStats: # To remove extra space before UID val = list(filter(None, values1)) # Merging extra columns created due to space in cmd ouput - val[8:] = [''.join(val[8:])] + val[8:] = [''.join(val[8:])] dict1 = dict(zip(keylist, val)) dict_list.append(dict1) return dict_list - + def update_dockerstats_command(self): data = self.run_command("docker stats --no-stream -a") dockerdata = self.format_docker_cmd_output(data) @@ -105,9 +105,8 @@ class ProcDockerStats: self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) pids = row.get('PIDS') self.update_state_db(value, 'PIDS', pids) - + def update_processstats_command(self): - data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -1024") processdata = self.format_process_cmd_output(data) value = "" @@ -133,7 +132,7 @@ class ProcDockerStats: self.update_state_db(value, 'TIME', time) cmd = row.get('CMD') self.update_state_db(value, 'CMD', cmd) - + def update_state_db(self, key1, key2, value2): self.state_db.set('STATE_DB', key1, key2, value2) From 57da086a713cb136dcfcec79fa873b44f27a2541 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 24 Oct 2019 15:41:43 -0700 Subject: [PATCH 07/21] adding seperate method for docker data formatting --- .../procdockerstats/procdockerstats | 128 ++++++++++++------ 1 file changed, 89 insertions(+), 39 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index d238317aba30..316d842daa1d 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -10,11 +10,13 @@ import time import syslog import subprocess import re -import swsssdk +import swsssdk from datetime import datetime VERSION = '1.0' + SYSLOG_IDENTIFIER = "procdockerstats" + REDIS_HOSTIP = "127.0.0.1" # ========================== Syslog wrappers ========================== @@ -38,13 +40,13 @@ class ProcDockerStats: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) - self.state_db.connect("STATE_DB") - + self.state_db.connect("STATE_DB") + def run_command(self, cmd): proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) (stdout, stderr) = proc.communicate() if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) + log_error("Error running command '{}'".format(cmd)) else: return stdout @@ -55,9 +57,10 @@ class ProcDockerStats: dict_list = [] for item in lines[1:]: values1 = re.split(" +", item) - dict1 = dict(zip(keys, values1)) + dict1 = dict(zip(keys, values1)) dict_list.append(dict1) - return dict_list + formatted_dict = self.create_docker_dict(dict_list) + return formatted_dict def format_process_cmd_output(self, cmdout): lines = re.split("\n", cmdout) @@ -70,44 +73,90 @@ class ProcDockerStats: # To remove extra space before UID val = list(filter(None, values1)) # Merging extra columns created due to space in cmd ouput - val[8:] = [''.join(val[8:])] + val[8:] = [''.join(val[8:])] dict1 = dict(zip(keylist, val)) dict_list.append(dict1) return dict_list - def update_dockerstats_command(self): - data = self.run_command("docker stats --no-stream -a") - dockerdata = self.format_docker_cmd_output(data) - value = "" - # wipe out all data before updating with new values - self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') - for row in dockerdata[0:]: + def convert_to_bytes(self, value): + u = re.search('[a-zA-Z]+', value) + v = float(filter(str.isdigit, value)) + unit = u.group(0) + b = 'B' + kb = 'KB' + mb = 'MB' + mib = 'MiB' + gib = 'GiB' + if unit.lower() == b.lower(): + return v + elif unit.lower() == kb.lower(): + v1 = v * 1000 + return v1 + elif unit.lower() == mb.lower(): + v1 = v * 1000000 + return v1 + elif unit.lower() == mib.lower(): + v1 = v * 1048576 + return v1 + elif unit.lower() == gib.lower(): + v1 = v * 1073741824 + return v1 + + def create_docker_dict(self, dict_list): + dockerdict = {} + for row in dict_list[0:]: cid = row.get('CONTAINER ID') if cid: - value = 'Docker_Stats|' + str(cid) - name = row.get('NAME') - self.update_state_db(value, 'NAME', name) - cpu = row.get('CPU %') - self.update_state_db(value, 'CPU%', str(cpu)) + key = 'Docker_Stats|' + str(cid) + dockerdict[key] = {} + dockerdict[key]['NAME'] = row.get('NAME') + + splitcol = row.get('CPU %') + cpu = re.split("%", str(splitcol)) + dockerdict[key]['CPU%'] = str(cpu[0]) + splitcol = row.get('MEM USAGE / LIMIT') memuse = re.split(" / ", str(splitcol)) - self.update_state_db(value, 'MEM', str(memuse[0])) - self.update_state_db(value, 'MEM_LIMIT', str(memuse[1])) - mem = row.get('MEM %') - self.update_state_db(value, 'MEM %', str(mem)) + # converting MiB and GiB to bytes + dockerdict[key]['MEM'] = str(self.convert_to_bytes(memuse[0])) + dockerdict[key]['MEM_LIMIT'] = str(self.convert_to_bytes(memuse[1])) + + splitcol = row.get('MEM %') + mem = re.split("%", str(splitcol)) + dockerdict[key]['MEM%'] = str(mem[0]) + splitcol = row.get('NET I/O') netio = re.split(" / ", str(splitcol)) - self.update_state_db(value, 'NET_IN', str(netio[0])) - self.update_state_db(value, 'NET_OUT', str(netio[1])) + dockerdict[key]['NET_IN'] = str(self.convert_to_bytes(netio[0])) + dockerdict[key]['NET_OUT'] = str(self.convert_to_bytes(netio[1])) + splitcol = row.get('BLOCK I/O') - blocio = re.split(" / ", str(splitcol)) - self.update_state_db(value, 'BLOCK_IN', str(blocio[0])) - self.update_state_db(value, 'BLOCK_OUT', str(blocio[1])) - pids = row.get('PIDS') - self.update_state_db(value, 'PIDS', pids) + blockio = re.split(" / ", str(splitcol)) + dockerdict[key]['BLOCK_IN'] = str(self.convert_to_bytes(blockio[0])) + dockerdict[key]['BLOCK_OUT'] = str(self.convert_to_bytes(blockio[1])) + + dockerdict[key]['PIDS'] = row.get('PIDS') + return dockerdict + + def update_dockerstats_command(self): + cmd = "docker stats --no-stream -a" + data = self.run_command(cmd) + if not data: + log_error("'{}' returned null output".format(cmd)) + return False + dockerdata = self.format_docker_cmd_output(data) + if not dockerdata: + log_error("formatting for docker output failed") + return False + # wipe out all data from state_db before updating + self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') + for k1,v1 in dockerdata.iteritems(): + for k2,v2 in v1.iteritems(): + self.update_state_db(k1, k2, v2) def update_processstats_command(self): - data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -1024") + + data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5") processdata = self.format_process_cmd_output(data) value = "" # wipe out all data before updating with new values @@ -146,16 +195,17 @@ class ProcDockerStats: # main start def main(): - log_info("process-docker stats aemon starting up..") - if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" - sys.exit(1) + log_info("process-docker stats daemon starting up..") + """if not os.getuid() == 0: + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" + sys.exit(1)""" pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds - time.sleep(120) - pd.run() + while True: + pd.run() + time.sleep(120) log_info("process-docker stats daemon exited") if __name__ == '__main__': - main() + main() \ No newline at end of file From 9119ba9dbbc654b88b04ea04bd07c45e5e1cbce4 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 24 Oct 2019 15:48:25 -0700 Subject: [PATCH 08/21] uncommenting root check --- files/image_config/procdockerstats/procdockerstats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 316d842daa1d..234de1d85d15 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -196,10 +196,10 @@ class ProcDockerStats: # main start def main(): log_info("process-docker stats daemon starting up..") - """if not os.getuid() == 0: - log_error("Must be root to run process-docker daemon") - print "Error: Must be root to run process-docker daemon" - sys.exit(1)""" + if not os.getuid() == 0: + log_error("Must be root to run process-docker daemon") + print "Error: Must be root to run process-docker daemon" + sys.exit(1) pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds while True: From cd49b304a867b4b59ab265b8490f1e475474757d Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 14:04:12 -0800 Subject: [PATCH 09/21] resolving comments on constant name, rounded integer --- .../procdockerstats/procdockerstats | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 234de1d85d15..5889c7b1e102 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -82,29 +82,29 @@ class ProcDockerStats: u = re.search('[a-zA-Z]+', value) v = float(filter(str.isdigit, value)) unit = u.group(0) - b = 'B' - kb = 'KB' - mb = 'MB' - mib = 'MiB' - gib = 'GiB' - if unit.lower() == b.lower(): + UNITS_B = 'B' + UNITS_KB = 'KB' + UNITS_MB = 'MB' + UNITS_MiB = 'MiB' + UNITS_GiB = 'GiB' + if unit.lower() == UNITS_B.lower(): return v - elif unit.lower() == kb.lower(): + elif unit.lower() == UNITS_KB.lower(): v1 = v * 1000 return v1 - elif unit.lower() == mb.lower(): - v1 = v * 1000000 + elif unit.lower() == UNITS_MB.lower(): + v1 = v * 1000 * 1000 return v1 - elif unit.lower() == mib.lower(): - v1 = v * 1048576 - return v1 - elif unit.lower() == gib.lower(): - v1 = v * 1073741824 + elif unit.lower() == UNITS_MiB.lower(): + v1 = v * 1024 * 1024 return v1 + elif unit.lower() == UNITS_GiB.lower(): + v1 = v * 1024 * 1024 * 1024 + return round(v1) def create_docker_dict(self, dict_list): dockerdict = {} - for row in dict_list[0:]: + for row in dict_list[0:]: cid = row.get('CONTAINER ID') if cid: key = 'Docker_Stats|' + str(cid) @@ -155,7 +155,6 @@ class ProcDockerStats: self.update_state_db(k1, k2, v2) def update_processstats_command(self): - data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5") processdata = self.format_process_cmd_output(data) value = "" From a56b356b7e0fa77375f902af2be7f253d1944256 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 14:58:18 -0800 Subject: [PATCH 10/21] Resolving comment for converting to integer and uppercase for table name in redis --- .../procdockerstats/procdockerstats | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 5889c7b1e102..c4a08e1b96e6 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -88,26 +88,26 @@ class ProcDockerStats: UNITS_MiB = 'MiB' UNITS_GiB = 'GiB' if unit.lower() == UNITS_B.lower(): - return v + return int(round(v)) elif unit.lower() == UNITS_KB.lower(): v1 = v * 1000 - return v1 + return int(round(v1)) elif unit.lower() == UNITS_MB.lower(): v1 = v * 1000 * 1000 - return v1 + return int(round(v1)) elif unit.lower() == UNITS_MiB.lower(): v1 = v * 1024 * 1024 - return v1 + return int(round(v1)) elif unit.lower() == UNITS_GiB.lower(): v1 = v * 1024 * 1024 * 1024 - return round(v1) + return int(round(v1)) def create_docker_dict(self, dict_list): dockerdict = {} for row in dict_list[0:]: cid = row.get('CONTAINER ID') if cid: - key = 'Docker_Stats|' + str(cid) + key = 'DOCKER_STATS|' + str(cid) dockerdict[key] = {} dockerdict[key]['NAME'] = row.get('NAME') @@ -149,7 +149,7 @@ class ProcDockerStats: log_error("formatting for docker output failed") return False # wipe out all data from state_db before updating - self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*') + self.state_db.delete_all_by_pattern('STATE_DB', 'DOCKER_STATS|*') for k1,v1 in dockerdata.iteritems(): for k2,v2 in v1.iteritems(): self.update_state_db(k1, k2, v2) @@ -159,11 +159,11 @@ class ProcDockerStats: processdata = self.format_process_cmd_output(data) value = "" # wipe out all data before updating with new values - self.state_db.delete_all_by_pattern('STATE_DB', 'Process_Stats|*') + self.state_db.delete_all_by_pattern('STATE_DB', 'PROCESS_STATS|*') for row in processdata[0:]: cid = row.get('PID') if cid: - value = 'Process_Stats|' + str(cid) + value = 'PROCESS_STATS|' + str(cid) uid = row.get('UID') self.update_state_db(value, 'UID', uid) ppid = row.get('PPID') From 8d0ef5466dc49498b3b8ebfa9d34c10c8f54a52d Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 15:03:26 -0800 Subject: [PATCH 11/21] adding restart=always --- files/image_config/procdockerstats/procdockerstats.service | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats.service b/files/image_config/procdockerstats/procdockerstats.service index 31169dd61058..78ee0fe3cd14 100644 --- a/files/image_config/procdockerstats/procdockerstats.service +++ b/files/image_config/procdockerstats/procdockerstats.service @@ -1,12 +1,12 @@ [Unit] -Description=Control Plane ACL configuration daemon +Description=Process and docker CPU/memory utilization data export daemon Requires=updategraph.service After=updategraph.service [Service] Type=simple ExecStart=/usr/bin/procdockerstats +Restart=Always [Install] -WantedBy=multi-user.target - +WantedBy=multi-user.target \ No newline at end of file From 0c51c2877177f11cfa229b00d93f5a91529fd762 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 15:56:12 -0800 Subject: [PATCH 12/21] adding entry in sonic_debian_extension.j2 adding more descriptive names line indented returning explicitly none --- .../build_templates/sonic_debian_extension.j2 | 5 ++ .../procdockerstats/procdockerstats | 47 ++++++++++--------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 0f27c15273c1..ed74dffcecad 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -240,6 +240,11 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT/etc/systemd/s echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/ +# Copy process docker cpu/memory utilization data export daemon +sudo cp $IMAGE_CONFIGS/procdockerstats/procdockerstats.service $FILESYSTEM_ROOT/etc/systemd/system/ +echo "procdockerstats.service" | sudo tee -a $GENERATED_SERVICE_FILE +sudo cp $IMAGE_CONFIGS/procdockerstats/procdockerstats $FILESYSTEM_ROOT/usr/bin/ + # Copy process-reboot-cause service files sudo cp $IMAGE_CONFIGS/process-reboot-cause/process-reboot-cause.service $FILESYSTEM_ROOT/etc/systemd/system/ echo "process-reboot-cause.service" | sudo tee -a $GENERATED_SERVICE_FILE diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index c4a08e1b96e6..59ce7b83d6d5 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -46,61 +46,62 @@ class ProcDockerStats: proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) (stdout, stderr) = proc.communicate() if proc.returncode != 0: - log_error("Error running command '{}'".format(cmd)) + log_error("Error running command '{}'".format(cmd)) + return None else: return stdout def format_docker_cmd_output(self, cmdout): lines = re.split("\n", cmdout) keys = re.split(" +", lines[0]) - dict1 = dict() - dict_list = [] + docker_data = dict() + docker_data_list = [] for item in lines[1:]: values1 = re.split(" +", item) - dict1 = dict(zip(keys, values1)) - dict_list.append(dict1) - formatted_dict = self.create_docker_dict(dict_list) + docker_data = dict(zip(keys, values1)) + docker_data_list.append(docker_data) + formatted_dict = self.create_docker_dict(docker_data_list) return formatted_dict def format_process_cmd_output(self, cmdout): lines = re.split("\n", cmdout) keys = re.split(" +", lines[0]) keylist = list(filter(None, keys)) - dict1 = dict() - dict_list = [] + process_data = dict() + process_data_list = [] for item in lines[1:]: values1 = re.split(" +", str(item)) # To remove extra space before UID val = list(filter(None, values1)) # Merging extra columns created due to space in cmd ouput val[8:] = [''.join(val[8:])] - dict1 = dict(zip(keylist, val)) - dict_list.append(dict1) - return dict_list + process_data = dict(zip(keylist, val)) + process_data_list.append(process_data) + return process_data_list def convert_to_bytes(self, value): - u = re.search('[a-zA-Z]+', value) - v = float(filter(str.isdigit, value)) - unit = u.group(0) + unit_value = re.search('[a-zA-Z]+', value) + value_to_convert = float(filter(str.isdigit, value)) + unit = unit_value.group(0) UNITS_B = 'B' UNITS_KB = 'KB' UNITS_MB = 'MB' UNITS_MiB = 'MiB' UNITS_GiB = 'GiB' if unit.lower() == UNITS_B.lower(): - return int(round(v)) + return int(round(value_to_convert)) elif unit.lower() == UNITS_KB.lower(): - v1 = v * 1000 - return int(round(v1)) + value_converted = value_to_convert * 1000 + return int(round(value_converted)) elif unit.lower() == UNITS_MB.lower(): - v1 = v * 1000 * 1000 - return int(round(v1)) + value_converted = value_to_convert * 1000 * 1000 + return int(round(value_converted)) elif unit.lower() == UNITS_MiB.lower(): - v1 = v * 1024 * 1024 - return int(round(v1)) + value_converted = value_to_convert * 1024 * 1024 + return int(round(value_converted)) elif unit.lower() == UNITS_GiB.lower(): - v1 = v * 1024 * 1024 * 1024 - return int(round(v1)) + value_converted = value_to_convert * 1024 * 1024 * 1024 + return int(round(value_converted)) def create_docker_dict(self, dict_list): dockerdict = {} From 147719f3e6405571149a0ff6abed4cbbe114ac78 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 16:34:27 -0800 Subject: [PATCH 13/21] added new line at end of file changed comment for daemon --- files/image_config/procdockerstats/procdockerstats | 9 +++++---- .../image_config/procdockerstats/procdockerstats.service | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 59ce7b83d6d5..6b7249432da2 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -1,7 +1,7 @@ # !/usr/bin/env python ''' -This code is for a specific daemon, process-docker. -Which sends process and docker CPU/memory utilization data to DB every 2 mins. +procdockerstats +Daemon which periodically gathers process and docker statistics and pushes the data to STATE_DB ''' import sys @@ -206,6 +206,7 @@ def main(): pd.run() time.sleep(120) log_info("process-docker stats daemon exited") - + if __name__ == '__main__': - main() \ No newline at end of file + main() + diff --git a/files/image_config/procdockerstats/procdockerstats.service b/files/image_config/procdockerstats/procdockerstats.service index 78ee0fe3cd14..a932d64e31f2 100644 --- a/files/image_config/procdockerstats/procdockerstats.service +++ b/files/image_config/procdockerstats/procdockerstats.service @@ -9,4 +9,5 @@ ExecStart=/usr/bin/procdockerstats Restart=Always [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target + From b00378df2f8b999563c893365d9f0792e08f2ef0 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 13 Nov 2019 17:40:03 -0800 Subject: [PATCH 14/21] uppercase for process and docker_stats rearrange imports based on comment --- .../image_config/procdockerstats/procdockerstats | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 6b7249432da2..c9a8c30eb0c0 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -4,14 +4,14 @@ procdockerstats Daemon which periodically gathers process and docker statistics and pushes the data to STATE_DB ''' -import sys import os -import time -import syslog -import subprocess import re -import swsssdk +import subprocess +import sys +import syslog +import time from datetime import datetime +import swsssdk VERSION = '1.0' @@ -154,6 +154,7 @@ class ProcDockerStats: for k1,v1 in dockerdata.iteritems(): for k2,v2 in v1.iteritems(): self.update_state_db(k1, k2, v2) + return True def update_processstats_command(self): data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5") @@ -189,9 +190,9 @@ class ProcDockerStats: self.update_dockerstats_command() datetimeobj = datetime.now() # Adding key to store latest update time. - self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj) + self.update_state_db('DOCKER_STATS|LastUpdateTime', 'lastupdate', datetimeobj) self.update_processstats_command() - self.update_state_db('Process_Stats|LastUpdateTime', 'lastupdate', datetimeobj) + self.update_state_db('PROCESS_STATS|LastUpdateTime', 'lastupdate', datetimeobj) # main start def main(): From 57c33ed72ab00b7ff803b8d9f6138972cf8d493e Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 14 Nov 2019 09:50:17 -0800 Subject: [PATCH 15/21] trying to add changes from previous commit --- files/image_config/procdockerstats/procdockerstats | 1 + 1 file changed, 1 insertion(+) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index c9a8c30eb0c0..7362e482fd90 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -11,6 +11,7 @@ import sys import syslog import time from datetime import datetime + import swsssdk VERSION = '1.0' From e2f147d3d4310420c0b0ef4a80a1cb5f239f05cb Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 14 Nov 2019 15:19:28 -0800 Subject: [PATCH 16/21] correcting return statement --- files/image_config/procdockerstats/procdockerstats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 7362e482fd90..5f0c425ce399 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -155,7 +155,7 @@ class ProcDockerStats: for k1,v1 in dockerdata.iteritems(): for k2,v2 in v1.iteritems(): self.update_state_db(k1, k2, v2) - return True + return True def update_processstats_command(self): data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5") From f0493114515ca3d9ed8aeb1468910da3fa4ab28f Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Fri, 15 Nov 2019 18:38:50 -0800 Subject: [PATCH 17/21] appending BYTES to converted columns --- files/image_config/procdockerstats/procdockerstats | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 5f0c425ce399..27dee4cb138d 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -120,8 +120,8 @@ class ProcDockerStats: splitcol = row.get('MEM USAGE / LIMIT') memuse = re.split(" / ", str(splitcol)) # converting MiB and GiB to bytes - dockerdict[key]['MEM'] = str(self.convert_to_bytes(memuse[0])) - dockerdict[key]['MEM_LIMIT'] = str(self.convert_to_bytes(memuse[1])) + dockerdict[key]['MEM_BYTES'] = str(self.convert_to_bytes(memuse[0])) + dockerdict[key]['MEM_LIMIT_BYTES'] = str(self.convert_to_bytes(memuse[1])) splitcol = row.get('MEM %') mem = re.split("%", str(splitcol)) @@ -129,13 +129,13 @@ class ProcDockerStats: splitcol = row.get('NET I/O') netio = re.split(" / ", str(splitcol)) - dockerdict[key]['NET_IN'] = str(self.convert_to_bytes(netio[0])) - dockerdict[key]['NET_OUT'] = str(self.convert_to_bytes(netio[1])) + dockerdict[key]['NET_IN_BYTES'] = str(self.convert_to_bytes(netio[0])) + dockerdict[key]['NET_OUT_BYTES'] = str(self.convert_to_bytes(netio[1])) splitcol = row.get('BLOCK I/O') blockio = re.split(" / ", str(splitcol)) - dockerdict[key]['BLOCK_IN'] = str(self.convert_to_bytes(blockio[0])) - dockerdict[key]['BLOCK_OUT'] = str(self.convert_to_bytes(blockio[1])) + dockerdict[key]['BLOCK_IN_BYTES'] = str(self.convert_to_bytes(blockio[0])) + dockerdict[key]['BLOCK_OUT_BYTES'] = str(self.convert_to_bytes(blockio[1])) dockerdict[key]['PIDS'] = row.get('PIDS') return dockerdict From cd006171d8dc0f2c444d7de655f5b1435c944bc6 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Wed, 20 Nov 2019 17:31:46 -0800 Subject: [PATCH 18/21] changing top 5 to 1024 processes --- files/image_config/procdockerstats/procdockerstats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstats/procdockerstats index 27dee4cb138d..738d69d49063 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstats/procdockerstats @@ -158,7 +158,7 @@ class ProcDockerStats: return True def update_processstats_command(self): - data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5") + data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -1024") processdata = self.format_process_cmd_output(data) value = "" # wipe out all data before updating with new values From 14a10563f63e99197d7778608d027305fde371ca Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 21 Nov 2019 14:09:04 -0800 Subject: [PATCH 19/21] adding d to suggest daemon file --- files/build_templates/sonic_debian_extension.j2 | 6 +++--- .../procdockerstats => procdockerstatsd/procdockerstatsd} | 8 ++++---- .../procdockerstatsd.service} | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) rename files/image_config/{procdockerstats/procdockerstats => procdockerstatsd/procdockerstatsd} (98%) rename files/image_config/{procdockerstats/procdockerstats.service => procdockerstatsd/procdockerstatsd.service} (85%) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index ed74dffcecad..1f731f1f7614 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -241,9 +241,9 @@ echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/ # Copy process docker cpu/memory utilization data export daemon -sudo cp $IMAGE_CONFIGS/procdockerstats/procdockerstats.service $FILESYSTEM_ROOT/etc/systemd/system/ -echo "procdockerstats.service" | sudo tee -a $GENERATED_SERVICE_FILE -sudo cp $IMAGE_CONFIGS/procdockerstats/procdockerstats $FILESYSTEM_ROOT/usr/bin/ +sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT/etc/systemd/system/ +echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE +sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd $FILESYSTEM_ROOT/usr/bin/ # Copy process-reboot-cause service files sudo cp $IMAGE_CONFIGS/process-reboot-cause/process-reboot-cause.service $FILESYSTEM_ROOT/etc/systemd/system/ diff --git a/files/image_config/procdockerstats/procdockerstats b/files/image_config/procdockerstatsd/procdockerstatsd similarity index 98% rename from files/image_config/procdockerstats/procdockerstats rename to files/image_config/procdockerstatsd/procdockerstatsd index 738d69d49063..4eb76593d472 100644 --- a/files/image_config/procdockerstats/procdockerstats +++ b/files/image_config/procdockerstatsd/procdockerstatsd @@ -1,6 +1,6 @@ # !/usr/bin/env python ''' -procdockerstats +procdockerstatsd Daemon which periodically gathers process and docker statistics and pushes the data to STATE_DB ''' @@ -16,7 +16,7 @@ import swsssdk VERSION = '1.0' -SYSLOG_IDENTIFIER = "procdockerstats" +SYSLOG_IDENTIFIER = "procdockerstatsd" REDIS_HOSTIP = "127.0.0.1" @@ -37,7 +37,7 @@ def log_error(msg, also_print_to_console=False): syslog.closelog() # ========================== ProcessDocker class ========================== -class ProcDockerStats: +class ProcDockerStatsd: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) @@ -202,7 +202,7 @@ def main(): log_error("Must be root to run process-docker daemon") print "Error: Must be root to run process-docker daemon" sys.exit(1) - pd = ProcDockerStats() + pd = ProcDockerStatsd() # Data need to be updated every 2 mins. hence adding delay of 120 seconds while True: pd.run() diff --git a/files/image_config/procdockerstats/procdockerstats.service b/files/image_config/procdockerstatsd/procdockerstatsd.service similarity index 85% rename from files/image_config/procdockerstats/procdockerstats.service rename to files/image_config/procdockerstatsd/procdockerstatsd.service index a932d64e31f2..c7ebee3b4a76 100644 --- a/files/image_config/procdockerstats/procdockerstats.service +++ b/files/image_config/procdockerstatsd/procdockerstatsd.service @@ -5,7 +5,7 @@ After=updategraph.service [Service] Type=simple -ExecStart=/usr/bin/procdockerstats +ExecStart=/usr/bin/procdockerstatsd Restart=Always [Install] From 4270c305caf9a972b0e75407c16dcd18c71a449e Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Thu, 21 Nov 2019 14:45:43 -0800 Subject: [PATCH 20/21] removing d from classname --- files/image_config/procdockerstatsd/procdockerstatsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/image_config/procdockerstatsd/procdockerstatsd b/files/image_config/procdockerstatsd/procdockerstatsd index 4eb76593d472..66d2d45009d5 100644 --- a/files/image_config/procdockerstatsd/procdockerstatsd +++ b/files/image_config/procdockerstatsd/procdockerstatsd @@ -37,7 +37,7 @@ def log_error(msg, also_print_to_console=False): syslog.closelog() # ========================== ProcessDocker class ========================== -class ProcDockerStatsd: +class ProcDockerStats: def __init__(self): self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP) @@ -202,7 +202,7 @@ def main(): log_error("Must be root to run process-docker daemon") print "Error: Must be root to run process-docker daemon" sys.exit(1) - pd = ProcDockerStatsd() + pd = ProcDockerStats() # Data need to be updated every 2 mins. hence adding delay of 120 seconds while True: pd.run() From eedab12bc9101620bea8fa4a3ec9b10b51403ca0 Mon Sep 17 00:00:00 2001 From: Pradnya Mohite Date: Tue, 26 Nov 2019 16:58:00 -0800 Subject: [PATCH 21/21] adding database.service as dependency adding process/docker in comment --- files/build_templates/sonic_debian_extension.j2 | 2 +- files/image_config/procdockerstatsd/procdockerstatsd.service | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 1f731f1f7614..fc0375362db8 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -240,7 +240,7 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT/etc/systemd/s echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/ -# Copy process docker cpu/memory utilization data export daemon +# Copy process/docker cpu/memory utilization data export daemon sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd.service $FILESYSTEM_ROOT/etc/systemd/system/ echo "procdockerstatsd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/procdockerstatsd/procdockerstatsd $FILESYSTEM_ROOT/usr/bin/ diff --git a/files/image_config/procdockerstatsd/procdockerstatsd.service b/files/image_config/procdockerstatsd/procdockerstatsd.service index c7ebee3b4a76..4e38c350a577 100644 --- a/files/image_config/procdockerstatsd/procdockerstatsd.service +++ b/files/image_config/procdockerstatsd/procdockerstatsd.service @@ -1,7 +1,7 @@ [Unit] Description=Process and docker CPU/memory utilization data export daemon -Requires=updategraph.service -After=updategraph.service +Requires=database.service updategraph.service +After=database.service updategraph.service [Service] Type=simple