Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[device/alibaba]: Update FPGA/CPLD upgrade logger #92

Merged
merged 2 commits into from Jun 6, 2019
Merged

[device/alibaba]: Update FPGA/CPLD upgrade logger #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ae65410
[device/alibaba]: Add cpld/fpga upgrade log
Jun 6, 2019
aaa6b6e
[device/alibaba]: Update get_last_upgrade_result to support cpld/fpga…
Jun 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 122 additions & 103 deletions device/alibaba/x86_64-alibaba_as13-32h-cl-r0/plugins/fwmgrutil.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import base64
import time
import json
import logging
import ast
from datetime import datetime

try:
Expand All @@ -29,7 +31,7 @@ def __init__(self):
self.bmc_raw_command_url = "http://240.1.1.1:8080/api/sys/raw"
self.fw_upgrade_url = "http://240.1.1.1:8080/api/sys/upgrade"
self.onie_config_file = "/host/machine.conf"
self.fw_upgrade_logger_path = "/usr/local/etc/last_fw_upgrade_logger"
self.fw_upgrade_logger_path = "/var/log/fw_upgrade.log"
self.cpldb_version_path = "/sys/devices/platform/%s.cpldb/getreg" % self.platform_name
self.fpga_version_path = "/sys/devices/platform/%s.switchboard/FPGA/getreg" % self.platform_name
self.switchboard_cpld1_path = "/sys/devices/platform/%s.switchboard/CPLD1/getreg" % self.platform_name
Expand Down Expand Up @@ -72,6 +74,24 @@ def __fpga_pci_rescan(self):
os.system('modprobe switchboard_fpga')
return 0

def __update_fw_upgrade_logger(self, header, message):
if not os.path.isfile(self.fw_upgrade_logger_path):
cmd = "sudo touch %s && sudo chmod +x %s" % (
self.fw_upgrade_logger_path, self.fw_upgrade_logger_path)
subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

logging.basicConfig(filename=self.fw_upgrade_logger_path,
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)

log_message = "%s : %s" % (header, message)
if header != "last_upgrade_result":
print(log_message)
return logging.info(log_message)

def get_bmc_pass(self):
if os.path.exists(self.bmc_pwd_path):
with open(self.bmc_pwd_path) as file:
Expand Down Expand Up @@ -487,15 +507,14 @@ def firmware_upgrade(self, fw_type, fw_path, fw_extra=None):
print("Failed: Invalid firmware type")
return False

self.upgrade_logger(upgrade_list)
return True

def get_last_upgrade_result(self):
"""
Get last firmware upgrade information, inlcudes:
1) FwType: cpld/fpga/bios/bmc(passed by method 'firmware_upgrade'), string
2) FwPath: path and file name of firmware(passed by method 'firmware_upgrade'), string
3) FwExtra: designated string, econdings of this string is determined by vendor(passed by method 'firmware_upgrade')
3) FwExtra: designated string, econdings of this string is determined by vendor(passed by method 'firmware_program')
4) Result: indicates whether the upgrade action is performed and success/failure status if performed. Values should be one of: "DONE"/"FAILED"/"NOT_PERFORMED".
list of object:
[
Expand All @@ -510,78 +529,29 @@ def get_last_upgrade_result(self):
"FwPath": "fan_cpld.vme"
"FwExtra": "FAN_CPLD"
"Result": "FAILED"
},
{
"FwType": "cpld",
"FwPath": "refresh_cpld.vme"
"FwExtra": "REFRESH_CPLD"
"Result": "DONE"
}
]
"""
last_update_list = []
local_log = []

if os.path.exists(self.fw_upgrade_logger_path):
with open(self.fw_upgrade_logger_path, 'r') as file:
data = file.read()

local_log = json.loads(data)
upgrade_info_req = requests.get(self.fw_upgrade_url)
if upgrade_info_req.status_code == 200 and 'success' in upgrade_info_req.json().get('result'):
upgrade_info_json = upgrade_info_req.json()

for info_json_key in upgrade_info_json.keys():
if info_json_key == "result" or upgrade_info_json[info_json_key] == "None":
continue

for x in range(0, len(upgrade_info_json[info_json_key])):
update_dict = dict()
index = x if info_json_key == "CPLD upgrade log" else -1
raw_data = upgrade_info_json[info_json_key][index]
raw_data_list = raw_data.split(",")
fw_path = raw_data_list[1].split("firmware:")[1].strip()
fw_extra_raw = raw_data_list[0].split(":")[0].strip()
fw_result_raw = raw_data_list[0].split(":")[1].strip()
raw_datetime = raw_data_list[2].split("time:")[1].strip()
reformat_datetime = raw_datetime.replace("CST", "UTC")
fw_extra_time = datetime.strptime(
reformat_datetime, '%a %b %d %H:%M:%S %Z %Y')
fw_extra_str = {
"top_lc": "TOP_LC_CPLD",
"bot_lc": "BOT_LC_CPLD",
"fan": "FAN_CPLD",
"cpu": "CPU_CPLD",
"base": "BASE_CPLD",
"combo": "COMBO_CPLD",
"switch": "SW_CPLD",
"enable": "REFRESH_CPLD",
}.get(fw_extra_raw, fw_extra_raw)
fw_result = "DONE" if fw_result_raw == "success" else fw_result_raw.upper()
fw_result = "FAILED" if "FAILED" in fw_result else fw_result
fw_result = "NOT_PERFORMED" if fw_result != "DONE" and fw_result != "FAILED" else fw_result
update_dict["FwType"] = info_json_key.split(" ")[0].lower()
update_dict["FwPath"] = fw_path
update_dict["FwExtra"] = fw_extra_str
update_dict["Result"] = fw_result
update_dict["FwTime"] = fw_extra_time
last_update_list.append(update_dict)
if info_json_key != "CPLD upgrade log":
break

last_update_list = sorted(last_update_list, key=lambda i: i['FwTime'])
map(lambda d: d.pop('FwTime', None), last_update_list)
if len(last_update_list) >= len(local_log) and len(last_update_list) != 0:
if str(last_update_list[-1].get('FwType')).lower() != 'cpld' or len(local_log) <= 1:
last_update_list = [last_update_list[-1]]
else:
remote_list = last_update_list[-len(local_log):]
remote_list_value = [d['FwPath'].lower()
for d in remote_list if 'FwPath' in d]
last_update_list = remote_list if remote_list_value == local_log else [
last_update_list[-1]]
elif len(last_update_list) != 0:
last_update_list = [last_update_list[-1]]
lines = file.read().splitlines()

upgrade_txt = [i for i in reversed(
lines) if "last_upgrade_result" in i]
if len(upgrade_txt) > 0:
last_upgrade_txt = upgrade_txt[0].split(
"last_upgrade_result : ")
last_upgrade_list = ast.literal_eval(last_upgrade_txt[1])
for x in range(0, len(last_upgrade_list[1].split(":"))):
upgrade_dict = {}
upgrade_dict["FwType"] = last_upgrade_list[0].lower()
upgrade_dict["FwPath"] = last_upgrade_list[1].split(":")[x]
upgrade_dict["FwExtra"] = last_upgrade_list[2].split(":")[
x] if last_upgrade_list[2] else "None"
upgrade_dict["Result"] = last_upgrade_list[3].split(":")[x]
last_update_list.append(upgrade_dict)

return last_update_list

Expand All @@ -604,14 +574,24 @@ def firmware_program(self, fw_type, fw_path, fw_extra=None):
self.firmware_program("FPGA", "/fpga.bin")
"""
fw_type = fw_type.lower()
upgrade_list = []
bmc_pwd = self.get_bmc_pass()
if not bmc_pwd and fw_type != "fpga":
print("Failed: BMC credential not found")
self.__update_fw_upgrade_logger(
"fw_upgrade", "fail, message=BMC credential not found")
return False

if fw_type == 'fpga':
print("FPGA Upgrade")
last_fw_upgrade = ["FPGA", fw_path, None, "FAILED"]
self.__update_fw_upgrade_logger(
"fpga_upgrade", "start FPGA upgrade")

if not os.path.isfile(fw_path):
self.__update_fw_upgrade_logger(
"fpga_upgrade", "fail, message=FPGA image not found %s" % fw_path)
self.__update_fw_upgrade_logger(
"last_upgrade_result", str(last_fw_upgrade))
return False

command = 'fpga_prog ' + fw_path
print("Running command: %s" % command)
process = subprocess.Popen(
Expand All @@ -621,12 +601,25 @@ def firmware_program(self, fw_type, fw_path, fw_extra=None):
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
print(output.strip())

rc = process.returncode
if rc != 0:
self.__update_fw_upgrade_logger(
"fw_upgrade", "fail, message=Unable to install FPGA")
self.__update_fw_upgrade_logger(
"last_upgrade_result", str(last_fw_upgrade))
return False

self.__update_fw_upgrade_logger("fpga_upgrade", "done")
last_fw_upgrade[3] = "DONE"
self.__update_fw_upgrade_logger(
"last_upgrade_result", str(last_fw_upgrade))
return True

elif 'cpld' in fw_type:
print("CPLD Upgrade")
self.__update_fw_upgrade_logger(
"cpld_upgrade", "start CPLD upgrade")

# Check input
fw_extra_str = str(fw_extra).upper()
if ":" in fw_path and ":" in fw_extra_str:
Expand All @@ -637,11 +630,12 @@ def firmware_program(self, fw_type, fw_path, fw_extra=None):
fw_extra_str_list = [fw_extra_str]

if len(fw_path_list) != len(fw_extra_str_list):
print("Failed: Invalid input")
self.__update_fw_upgrade_logger(
"cpld_upgrade", "fail, message=Invalid input")
return False

cpld_result_list = []
data_list = list(zip(fw_path_list, fw_extra_str_list))
pass_count = 0
for data in data_list:
fw_path = data[0]
fw_extra_str = data[1]
Expand All @@ -658,25 +652,31 @@ def firmware_program(self, fw_type, fw_path, fw_extra=None):
"SW_CPLD2": "switch"
}.get(fw_extra_str, None)

print("Upgrade %s cpld" % fw_extra_str)
self.__update_fw_upgrade_logger(
"cpld_upgrade", "start %s upgrade" % data[1])
upgrade_result = "FAILED"
for x in range(1, 4):
# Set fw_extra
if x > 1:
print("Retry to upgrade %s" % data[1])
self.__update_fw_upgrade_logger(
"cpld_upgrade", "fail, message=Retry to upgrade %s" % data[1])

if fw_extra_str is None:
print(
"Failed: Invalid extra information string %s" % data[1])
elif fw_extra_str is None:
self.__update_fw_upgrade_logger(
"cpld_upgrade", "fail, message=Invalid extra information string %s" % data[1])
break
elif not os.path.isfile(os.path.abspath(fw_path)):
self.__update_fw_upgrade_logger(
"cpld_upgrade", "fail, message=CPLD image not found %s" % fw_path)
break

filename_w_ext = os.path.basename(fw_path)
# Install cpld image via ispvm tool
print("Installing...")
command = 'ispvm %s' % fw_path
if fw_extra_str in ["top_lc", "bottom_lc"]:
option = 1 if fw_extra_str == "top_lc" else 2
command = "ispvm -c %d %s" % (option,
os.path.abspath(fw_path))
os.path.abspath(fw_path))
print("Running command : %s" % command)
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Expand All @@ -688,22 +688,25 @@ def firmware_program(self, fw_type, fw_path, fw_extra=None):

rc = process.returncode
if rc != 0:
print("Failed: Unable to install CPLD")
self.__update_fw_upgrade_logger(
"cpld_upgrade", "fail, message=Unable to install CPLD")
continue

print("%s upgrade completed\n" % fw_extra_str)
pass_count += 1
upgrade_list.append(filename_w_ext.lower())
upgrade_result = "DONE"
self.__update_fw_upgrade_logger("cpld_upgrade", "done")
break
cpld_result_list.append(upgrade_result)

self.upgrade_logger(upgrade_list)
print("Done")
return pass_count == len(data_list)
last_fw_upgrade = ["CPLD", ":".join(
fw_path_list), ":".join(fw_extra_str_list), ":".join(cpld_result_list)]
self.__update_fw_upgrade_logger(
"last_upgrade_result", str(last_fw_upgrade))
return "FAILED" not in cpld_result_list
else:
print("Failed: Invalid firmware type")
self.__update_fw_upgrade_logger(
"fw_upgrade", "fail, message=Invalid firmware type")
return False

self.upgrade_logger(upgrade_list)
return True

def firmware_refresh(self, fpga_list, cpld_list, fw_extra=None):
Expand Down Expand Up @@ -731,10 +734,13 @@ def firmware_refresh(self, fpga_list, cpld_list, fw_extra=None):
return False

if cpld_list and ("BASE_CPLD" in cpld_list or "FAN_CPLD" in cpld_list):
self.__update_fw_upgrade_logger(
"cpld_refresh", "start cpld refresh")
fw_path_list = fw_extra.split(':')
command = "echo "
if len(fw_path_list) != len(cpld_list):
print("Failed: Invalid fw_extra")
self.__update_fw_upgrade_logger(
"cpld_refresh", "fail, message=Invalid fw_extra")
return False

for idx in range(0, len(cpld_list)):
Expand All @@ -747,12 +753,12 @@ def firmware_refresh(self, fpga_list, cpld_list, fw_extra=None):
if not refresh_type:
continue
elif not self.upload_file_bmc(fw_path):
print("Failed: Unable to upload refresh image to BMC")
self.__update_fw_upgrade_logger(
"cpld_refresh", "fail, message=Unable to upload refresh image to BMC")
return False
else:
filename_w_ext = os.path.basename(fw_path)
upgrade_list.append(filename_w_ext.lower())
self.upgrade_logger(upgrade_list)

sub_command = "%s /home/root/%s > /tmp/cpld_refresh " % (
refresh_type, filename_w_ext)
Expand All @@ -762,25 +768,38 @@ def firmware_refresh(self, fpga_list, cpld_list, fw_extra=None):
json_data["data"] = command
r = requests.post(self.bmc_raw_command_url, json=json_data)
if r.status_code != 200:
print("Failed: %d Invalid refresh image" % r.status_code)
self.__update_fw_upgrade_logger(
"cpld_refresh", "fail, message=%d Invalid refresh image" % r.status_code)
return False
self.__update_fw_upgrade_logger("cpld_refresh", "done")

elif fpga_list:
elif cpld_list:
self.__update_fw_upgrade_logger(
"cpld_refresh", "start CPLD refresh")
json_data = dict()
json_data["data"] = "echo fpga > /tmp/cpld_refresh"
json_data["data"] = "echo cpu_cpld > /tmp/cpld_refresh"
r = requests.post(self.bmc_raw_command_url, json=json_data)
if r.status_code != 200:
print("Failed: %d Unable to load new FPGA" % r.status_code)
self.__update_fw_upgrade_logger(
"cpld_refresh", "fail, message=%d Unable to load new FPGA" % r.status_code)
return False
elif cpld_list:
self.__update_fw_upgrade_logger("cpld_refresh", "done")

elif fpga_list:
self.__update_fw_upgrade_logger(
"fpga_refresh", "start FPGA refresh")
json_data = dict()
json_data["data"] = "echo cpu_cpld > /tmp/cpld_refresh"
json_data["data"] = "echo fpga > /tmp/cpld_refresh"
r = requests.post(self.bmc_raw_command_url, json=json_data)
if r.status_code != 200:
print("Failed: %d Unable to load new CPLD" % r.status_code)
self.__update_fw_upgrade_logger(
"cpld_refresh", "fail, message=%d Unable to load new FPGA" % r.status_code)
return False
self.__update_fw_upgrade_logger("fpga_refresh", "done")

else:
print("Failed: Invalid input")
self.__update_fw_upgrade_logger(
"fw_refresh", "fail, message=Invalid input")
return False

return True
Expand Down
Loading