From 60e54109d1f28c1d5ea363436be65b4bf3272fd5 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 2 Jul 2020 16:21:25 -0700 Subject: [PATCH] [config] Add ConfigMgmt class for config validation, delete ports, add ports (#765) Provided a new ConfigMgmt class for - Config Validation - Adding ports - Deleting ports Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 840 ++++++++++++++++++ sonic-utilities-tests/config_mgmt_test.py | 721 +++++++++++++++ .../mock_tables/counters_db.json | 6 + 3 files changed, 1567 insertions(+) create mode 100644 config/config_mgmt.py create mode 100644 sonic-utilities-tests/config_mgmt_test.py diff --git a/config/config_mgmt.py b/config/config_mgmt.py new file mode 100644 index 0000000000..c9db79ea90 --- /dev/null +++ b/config/config_mgmt.py @@ -0,0 +1,840 @@ +''' +config_mgmt.py provides classes for configuration validation and for Dynamic +Port Breakout. +''' +try: + import re + import syslog + + from json import load + from time import sleep as tsleep + from imp import load_source + from jsondiff import diff + from sys import flags + + # SONiC specific imports + import sonic_yang + from swsssdk import ConfigDBConnector, SonicV2Connector, port_util + + # Using load_source to 'import /usr/local/bin/sonic-cfggen as sonic_cfggen' + # since /usr/local/bin/sonic-cfggen does not have .py extension. + load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') + from sonic_cfggen import deep_update, FormatConverter, sort_data + +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +# Globals +YANG_DIR = "/usr/local/yang-models" +CONFIG_DB_JSON_FILE = '/etc/sonic/confib_db.json' +# TODO: Find a place for it on sonic switch. +DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/port_breakout_config_db.json' + +class ConfigMgmt(): + ''' + Class to handle config managment for SONIC, this class will use sonic_yang + to verify config for the commands which are capable of change in config DB. + ''' + + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): + ''' + Initialise the class, --read the config, --load in data tree. + + Parameters: + source (str): source for input config, default configDb else file. + debug (bool): verbose mode. + allowTablesWithoutYang (bool): allow tables without yang model in + config or not. + + Returns: + void + ''' + try: + self.configdbJsonIn = None + self.configdbJsonOut = None + self.allowTablesWithoutYang = allowTablesWithoutYang + + # logging vars + self.SYSLOG_IDENTIFIER = "ConfigMgmt" + self.DEBUG = debug + + self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + # load yang models + self.sy.loadYangModel() + # load jIn from config DB or from config DB json file. + if source.lower() == 'configdb': + self.readConfigDB() + # treat any other source as file input + else: + self.readConfigDBJson(source) + # this will crop config, xlate and load. + self.sy.loadData(self.configdbJsonIn) + + # Raise if tables without YANG models are not allowed but exist. + if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang): + raise Exception('Config has tables without YANG models') + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise(Exception('ConfigMgmt Class creation failed')) + + return + + def __del__(self): + pass + + def tablesWithoutYang(self): + ''' + Return tables loaded in config for which YANG model does not exist. + + Parameters: + void + + Returns: + tablesWithoutYang (list): list of tables. + ''' + return self.sy.tablesWithOutYang + + def loadData(self, configdbJson): + ''' + Explicit function to load config data in Yang Data Tree. + + Parameters: + configdbJson (dict): dict similar to configDb. + + Returns: + void + ''' + self.sy.loadData(configdbJson) + # Raise if tables without YANG models are not allowed but exist. + if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang): + raise Exception('Config has tables without YANG models') + + return + + def validateConfigData(self): + ''' + Validate current config data Tree. + + Parameters: + void + + Returns: + bool + ''' + try: + self.sy.validate_data_tree() + except Exception as e: + self.sysLog(msg='Data Validation Failed') + return False + + self.sysLog(msg='Data Validation successful', doPrint=True) + return True + + def sysLog(self, logLevel=syslog.LOG_INFO, msg=None, doPrint=False): + ''' + Log the msg in syslog file. + + Parameters: + debug : syslog level + msg (str): msg to be logged. + + Returns: + void + ''' + # log debug only if enabled + if self.DEBUG == False and logLevel == syslog.LOG_DEBUG: + return + if flags.interactive !=0 and doPrint == True: + print("{}".format(msg)) + syslog.openlog(self.SYSLOG_IDENTIFIER) + syslog.syslog(logLevel, msg) + syslog.closelog() + + return + + def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): + ''' + Read the config from a Config File. + + Parameters: + source(str): config file name. + + Returns: + (void) + ''' + self.sysLog(msg='Reading data from {}'.format(source)) + self.configdbJsonIn = readJsonFile(source) + #self.sysLog(msg=type(self.configdbJsonIn)) + if not self.configdbJsonIn: + raise(Exception("Can not load config from config DB json file")) + self.sysLog(msg='Reading Input {}'.format(self.configdbJsonIn)) + + return + + """ + Get config from redis config DB + """ + def readConfigDB(self): + ''' + Read the config in Config DB. Assign it in self.configdbJsonIn. + + Parameters: + (void) + + Returns: + (void) + ''' + self.sysLog(doPrint=True, msg='Reading data from Redis configDb') + # Read from config DB on sonic switch + db_kwargs = dict(); data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect() + deep_update(data, FormatConverter.db_to_output(configdb.get_config())) + self.configdbJsonIn = FormatConverter.to_serialized(data) + self.sysLog(syslog.LOG_DEBUG, 'Reading Input from ConfigDB {}'.\ + format(self.configdbJsonIn)) + + return + + def writeConfigDB(self, jDiff): + ''' + Write the diff in Config DB. + + Parameters: + jDiff (dict): config to push in config DB. + + Returns: + void + ''' + self.sysLog(doPrint=True, msg='Writing in Config DB') + db_kwargs = dict(); data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect(False) + deep_update(data, FormatConverter.to_deserialized(jDiff)) + data = sort_data(data) + self.sysLog(msg="Write in DB: {}".format(data)) + configdb.mod_config(FormatConverter.output_to_db(data)) + + return + +# End of Class ConfigMgmt + +class ConfigMgmtDPB(ConfigMgmt): + ''' + Config MGMT class for Dynamic Port Breakout(DPB). This is derived from + ConfigMgmt. + ''' + + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): + ''' + Initialise the class + + Parameters: + source (str): source for input config, default configDb else file. + debug (bool): verbose mode. + allowTablesWithoutYang (bool): allow tables without yang model in + config or not. + + Returns: + void + ''' + try: + ConfigMgmt.__init__(self, source=source, debug=debug, \ + allowTablesWithoutYang=allowTablesWithoutYang) + self.oidKey = 'ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x' + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise(Exception('ConfigMgmtDPB Class creation failed')) + + return + + def __del__(self): + pass + + def _checkKeyinAsicDB(self, key, db): + ''' + Check if a key exists in ASIC DB or not. + + Parameters: + db (SonicV2Connector): database. + key (str): key in ASIC DB, with table Seperator if applicable. + + Returns: + (bool): True, if given key is present. + ''' + self.sysLog(msg='Check Key in Asic DB: {}'.format(key)) + try: + # chk key in ASIC DB + if db.exists('ASIC_DB', key): + return True + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise(e) + + return False + + def _checkNoPortsInAsicDb(self, db, ports, portMap): + ''' + Check ASIC DB for PORTs in port List + + Parameters: + db (SonicV2Connector): database. + ports (list): List of ports + portMap (dict): port to OID map. + + Returns: + (bool): True, if all ports are not present. + ''' + try: + # connect to ASIC DB, + db.connect(db.ASIC_DB) + for port in ports: + key = self.oidKey + portMap[port] + if self._checkKeyinAsicDB(key, db) == True: + return False + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + return False + + return True + + def _verifyAsicDB(self, db, ports, portMap, timeout): + ''' + Verify in the Asic DB that port are deleted, Keep on trying till timeout + period. + + Parameters: + db (SonicV2Connector): database. + ports (list): port list to check in ASIC DB. + portMap (dict): oid<->port map. + timeout (int): timeout period + + Returns: + (bool) + ''' + self.sysLog(doPrint=True, msg="Verify Port Deletion from Asic DB, Wait...") + try: + for waitTime in range(timeout): + self.sysLog(logLevel=syslog.LOG_DEBUG, msg='Check Asic DB: {} \ + try'.format(waitTime+1)) + # checkNoPortsInAsicDb will return True if all ports are not + # present in ASIC DB + if self._checkNoPortsInAsicDb(db, ports, portMap): + break + tsleep(1) + + # raise if timer expired + if waitTime + 1 == timeout: + self.sysLog(syslog.LOG_CRIT, "!!! Critical Failure, Ports \ + are not Deleted from ASIC DB, Bail Out !!!", doPrint=True) + raise(Exception("Ports are present in ASIC DB after {} secs".\ + format(timeout))) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise e + + return True + + def breakOutPort(self, delPorts=list(), portJson=dict(), force=False, \ + loadDefConfig=True): + ''' + This is the main function for port breakout. Exposed to caller. + + Parameters: + delPorts (list): ports to be deleted. + portJson (dict): Config DB json Part of all Ports, generated from + platform.json. + force (bool): if false return dependecies, else delete dependencies. + loadDefConfig: If loadDefConfig, add default config for ports as well. + + Returns: + (deps, ret) (tuple)[list, bool]: dependecies and success/failure. + ''' + MAX_WAIT = 60 + try: + # delete Port and get the Config diff, deps and True/False + delConfigToLoad, deps, ret = self._deletePorts(ports=delPorts, \ + force=force) + # return dependencies if delete port fails + if ret == False: + return deps, ret + + # add Ports and get the config diff and True/False + addConfigtoLoad, ret = self._addPorts(portJson=portJson, \ + loadDefConfig=loadDefConfig) + # return if ret is False, Great thing, no change is done in Config + if ret == False: + return None, ret + + # Save Port OIDs Mapping Before Deleting Port + dataBase = SonicV2Connector(host="127.0.0.1") + if_name_map, if_oid_map = port_util.get_interface_oid_map(dataBase) + self.sysLog(syslog.LOG_DEBUG, 'if_name_map {}'.format(if_name_map)) + + # If we are here, then get ready to update the Config DB, Update + # deletion of Config first, then verify in Asic DB for port deletion, + # then update addition of ports in config DB. + self.writeConfigDB(delConfigToLoad) + # Verify in Asic DB, + self._verifyAsicDB(db=dataBase, ports=delPorts, portMap=if_name_map, \ + timeout=MAX_WAIT) + self.writeConfigDB(addConfigtoLoad) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + return None, False + + return None, True + + def _deletePorts(self, ports=list(), force=False): + ''' + Delete ports and dependecies from data tree, validate and return resultant + config. + + Parameters: + ports (list): list of ports + force (bool): if false return dependecies, else delete dependencies. + + Returns: + (configToLoad, deps, ret) (tuple)[dict, list, bool]: config, dependecies + and success/fail. + ''' + configToLoad = None; deps = None + try: + self.sysLog(msg="delPorts ports:{} force:{}".format(ports, force)) + + self.sysLog(doPrint=True, msg='Start Port Deletion') + deps = list() + + # Get all dependecies for ports + for port in ports: + xPathPort = self.sy.findXpathPortLeaf(port) + self.sysLog(doPrint=True, msg='Find dependecies for port {}'.\ + format(port)) + dep = self.sy.find_data_dependencies(str(xPathPort)) + if dep: + deps.extend(dep) + + # No further action with no force and deps exist + if force == False and deps: + return configToLoad, deps, False; + + # delets all deps, No topological sort is needed as of now, if deletion + # of deps fails, return immediately + elif deps: + for dep in deps: + self.sysLog(msg='Deleting {}'.format(dep)) + self.sy.deleteNode(str(dep)) + # mark deps as None now, + deps = None + + # all deps are deleted now, delete all ports now + for port in ports: + xPathPort = self.sy.findXpathPort(port) + self.sysLog(doPrint=True, msg="Deleting Port: " + port) + self.sy.deleteNode(str(xPathPort)) + + # Let`s Validate the tree now + if self.validateConfigData()==False: + return configToLoad, deps, False; + + # All great if we are here, Lets get the diff + self.configdbJsonOut = self.sy.getData() + # Update configToLoad + configToLoad = self._updateDiffConfigDB() + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Port Deletion Failed") + return configToLoad, deps, False + + return configToLoad, deps, True + + def _addPorts(self, portJson=dict(), loadDefConfig=True): + ''' + Add ports and default confug in data tree, validate and return resultant + config. + + Parameters: + portJson (dict): Config DB json Part of all Ports, generated from + platform.json. + loadDefConfig: If loadDefConfig, add default config for ports as well. + + Returns: + (configToLoad, ret) (tuple)[dict, bool] + ''' + configToLoad = None + ports = portJson['PORT'].keys() + try: + self.sysLog(doPrint=True, msg='Start Port Addition') + self.sysLog(msg="addPorts Args portjson: {} loadDefConfig: {}".\ + format(portJson, loadDefConfig)) + + if loadDefConfig: + defConfig = self._getDefaultConfig(ports) + self.sysLog(msg='Default Config: {}'.format(defConfig)) + + # get the latest Data Tree, save this in input config, since this + # is our starting point now + self.configdbJsonIn = self.sy.getData() + + # Get the out dict as well, if not done already + if self.configdbJsonOut is None: + self.configdbJsonOut = self.sy.getData() + + # update portJson in configdbJsonOut PORT part + self.configdbJsonOut['PORT'].update(portJson['PORT']) + # merge new config with data tree, this is json level merge. + # We do not allow new table merge while adding default config. + if loadDefConfig: + self.sysLog(doPrint=True, msg="Merge Default Config for {}".\ + format(ports)) + self._mergeConfigs(self.configdbJsonOut, defConfig, True) + + # create a tree with merged config and validate, if validation is + # sucessful, then configdbJsonOut contains final and valid config. + self.sy.loadData(self.configdbJsonOut) + if self.validateConfigData()==False: + return configToLoad, False + + # All great if we are here, Let`s get the diff and update COnfig + configToLoad = self._updateDiffConfigDB() + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Port Addition Failed") + return configToLoad, False + + return configToLoad, True + + def _mergeConfigs(self, D1, D2, uniqueKeys=True): + ''' + Merge D2 dict in D1 dict, Note both first and second dict will change. + First Dict will have merged part D1 + D2. Second dict will have D2 - D1 + i.e [unique keys in D2]. Unique keys in D2 will be merged in D1 only + if uniqueKeys=True. + Usage: This function can be used with 'config load' command to merge + new config with old. + + Parameters: + D1 (dict): Partial Config 1. + D2 (dict): Partial Config 2. + uniqueKeys (bool) + + Returns: + bool + ''' + try: + def _mergeItems(it1, it2): + if isinstance(it1, list) and isinstance(it2, list): + it1.extend(it2) + elif isinstance(it1, dict) and isinstance(it2, dict): + self._mergeConfigs(it1, it2) + elif isinstance(it1, list) or isinstance(it2, list): + raise Exception("Can not merge Configs, List problem") + elif isinstance(it1, dict) or isinstance(it2, dict): + raise Exception("Can not merge Configs, Dict problem") + else: + # First Dict takes priority + pass + return + + for it in D1.keys(): + # D2 has the key + if D2.get(it): + _mergeItems(D1[it], D2[it]) + del D2[it] + + # if uniqueKeys are needed, merge rest of the keys of D2 in D1 + if uniqueKeys: + D1.update(D2) + except Exce as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Merge Config failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise e + + return D1 + + def _searchKeysInConfig(self, In, Out, skeys): + ''' + Search Relevant Keys in Input Config using DFS, This function is mainly + used to search ports related config in Default ConfigDbJson file. + + Parameters: + In (dict): Input Config to be searched + skeys (list): Keys to be searched in Input Config i.e. search Keys. + Out (dict): Contains the search result, i.e. Output Config with skeys. + + Returns: + found (bool): True if any of skeys is found else False. + ''' + found = False + if isinstance(In, dict): + for key in In.keys(): + for skey in skeys: + # pattern is very specific to current primary keys in + # config DB, may need to be updated later. + pattern = '^' + skey + '\|' + '|' + skey + '$' + \ + '|' + '^' + skey + '$' + reg = re.compile(pattern) + if reg.search(key): + # In primary key, only 1 match can be found, so return + Out[key] = In[key] + found = True + break + # Put the key in Out by default, if not added already. + # Remove later, if subelements does not contain any port. + if Out.get(key) is None: + Out[key] = type(In[key])() + if self._searchKeysInConfig(In[key], Out[key], skeys) == False: + del Out[key] + else: + found = True + + elif isinstance(In, list): + for skey in skeys: + if skey in In: + found = True + Out.append(skey) + + else: + # nothing for other keys + pass + + return found + + def configWithKeys(self, configIn=dict(), keys=list()): + ''' + This function returns the config with relavant keys in Input Config. + It calls _searchKeysInConfig. + + Parameters: + configIn (dict): Input Config + keys (list): Key list. + + Returns: + configOut (dict): Output Config containing only key related config. + ''' + configOut = dict() + try: + if len(configIn) and len(keys): + self._searchKeysInConfig(configIn, configOut, skeys=keys) + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="configWithKeys Failed, Error: {}".format(str(e))) + raise e + + return configOut + + def _getDefaultConfig(self, ports=list()): + ''' + Create a default Config for given Port list from Default Config File. + It calls _searchKeysInConfig. + + Parameters: + ports (list): list of ports, for which default config must be fetched. + + Returns: + defConfigOut (dict): default Config for given Ports. + ''' + # function code + try: + self.sysLog(doPrint=True, msg="Generating default config for {}".format(ports)) + defConfigIn = readJsonFile(DEFAULT_CONFIG_DB_JSON_FILE) + defConfigOut = dict() + self._searchKeysInConfig(defConfigIn, defConfigOut, skeys=ports) + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="getDefaultConfig Failed, Error: {}".format(str(e))) + raise e + + return defConfigOut + + def _updateDiffConfigDB(self): + ''' + Return ConfigDb format Diff b/w self.configdbJsonIn, self.configdbJsonOut + + Parameters: + void + + Returns: + configToLoad (dict): ConfigDb format Diff + ''' + try: + # Get the Diff + self.sysLog(msg='Generate Final Config to write in DB') + configDBdiff = self._diffJson() + # Process diff and create Config which can be updated in Config DB + configToLoad = self._createConfigToLoad(configDBdiff, \ + self.configdbJsonIn, self.configdbJsonOut) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Config Diff Generation failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise e + + return configToLoad + + def _createConfigToLoad(self, diff, inp, outp): + ''' + Create the config to write in Config DB, i.e. compitible with mod_config() + This functions has 3 inner functions: + -- _deleteHandler: to handle delete in diff. See example below. + -- _insertHandler: to handle insert in diff. See example below. + -- _recurCreateConfig: recursively create this config. + + Parameters: + diff: jsondiff b/w 2 configs. + Example: + {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}}, + u'Vlan777': {u'members': {insert: [(92, 'Ethernet2')]}}}, + 'PORT': {delete: {u'Ethernet1': {...}}}} + + inp: input config before delete/add ports, i.e. current config Db. + outp: output config after delete/add ports. i.e. config DB once diff + is applied. + + Returns: + configToLoad (dict): config in a format compitible with mod_Config(). + ''' + + ### Internal Functions ### + def _deleteHandler(diff, inp, outp, config): + ''' + Handle deletions in diff dict + ''' + if isinstance(inp, dict): + # Example Case: diff = PORT': {delete: {u'Ethernet1': {...}}}} + for key in diff: + # make sure keys from diff are present in inp but not in outp + if key in inp and key not in outp: + # assign key to None(null), redis will delete entire key + config[key] = None + else: + # should not happen + raise Exception('Invalid deletion of {} in diff'.format(key)) + + elif isinstance(inp, list): + # Example case: {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}} + # just take list from outputs + config.extend(outp) + return + + def _insertHandler(diff, inp, outp, config): + ''' + Handle inserts in diff dict + ''' + if isinstance(outp, dict): + # Example Case: diff = PORT': {insert: {u'Ethernet1': {...}}}} + for key in diff: + # make sure keys are only in outp + if key not in inp and key in outp: + # assign key in config same as outp + config[key] = outp[key] + else: + # should not happen + raise Exception('Invalid insertion of {} in diff'.format(key)) + + elif isinstance(outp, list): + # just take list from output + # Example case: {u'VLAN': {u'Vlan100': {'members': {insert: [(95, 'Ethernet1')]}} + config.extend(outp) + return + + def _recurCreateConfig(diff, inp, outp, config): + ''' + Recursively iterate diff to generate config to write in configDB + ''' + changed = False + # updates are represented by list in diff and as dict in outp\inp + # we do not allow updates right now + if isinstance(diff, list) and isinstance(outp, dict): + return changed + + idx = -1 + for key in diff: + idx = idx + 1 + if str(key) == '$delete': + _deleteHandler(diff[key], inp, outp, config) + changed = True + elif str(key) == '$insert': + _insertHandler(diff[key], inp, outp, config) + changed = True + else: + # insert in config by default, remove later if not needed + if isinstance(diff, dict): + # config should match type of outp + config[key] = type(outp[key])() + if _recurCreateConfig(diff[key], inp[key], outp[key], \ + config[key]) == False: + del config[key] + else: + changed = True + elif isinstance(diff, list): + config.append(key) + if _recurCreateConfig(diff[idx], inp[idx], outp[idx], \ + config[-1]) == False: + del config[-1] + else: + changed = True + + return changed + + ### Function Code ### + try: + configToLoad = dict() + _recurCreateConfig(diff, inp, outp, configToLoad) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Create Config to load in DB, Failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise e + + return configToLoad + + def _diffJson(self): + ''' + Return json diff between self.configdbJsonIn, self.configdbJsonOut dicts. + + Parameters: + void + + Returns: + (dict): json diff between self.configdbJsonIn, self.configdbJsonOut + dicts. + Example: + {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}}, + u'Vlan777': {u'members': {insert: [(92, 'Ethernet2')]}}}, + 'PORT': {delete: {u'Ethernet1': {...}}}} + ''' + return diff(self.configdbJsonIn, self.configdbJsonOut, syntax='symmetric') + +# end of class ConfigMgmtDPB + +# Helper Functions +def readJsonFile(fileName): + ''' + Read Json file. + + Parameters: + fileName (str): file + + Returns: + result (dict): json --> dict + ''' + try: + with open(fileName) as f: + result = load(f) + except Exception as e: + raise Exception(e) + + return result diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py new file mode 100644 index 0000000000..aec7f75e30 --- /dev/null +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -0,0 +1,721 @@ +import imp +import os +# import file under test i.e. config_mgmt.py +imp.load_source('config_mgmt', \ + os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')) +import config_mgmt + +from unittest import TestCase +from mock import MagicMock, call +from json import dump + +class TestConfigMgmt(TestCase): + ''' + Test Class for config_mgmt.py + ''' + + def setUp(self): + config_mgmt.CONFIG_DB_JSON_FILE = "startConfigDb.json" + config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE = "portBreakOutConfigDb.json" + return + + def test_config_validation(self): + curConfig = dict(configDbJson) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) + assert cm.validateConfigData() == True + return + + def test_table_without_yang(self): + curConfig = dict(configDbJson) + unknown = {"unknown_table": {"ukey": "uvalue"}} + self.updateConfig(curConfig, unknown) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) + #assert "unknown_table" in cm.tablesWithoutYang() + return + + def test_search_keys(self): + curConfig = dict(configDbJson) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) + out = cmdpb.configWithKeys(portBreakOutConfigDbJson, \ + ["Ethernet8","Ethernet9"]) + assert "VLAN" not in out.keys() + assert "INTERFACE" not in out.keys() + for k in out['ACL_TABLE'].keys(): + # only ports must be chosen + len(out['ACL_TABLE'][k]) == 1 + out = cmdpb.configWithKeys(portBreakOutConfigDbJson, \ + ["Ethernet10","Ethernet11"]) + assert "INTERFACE" in out.keys() + for k in out['ACL_TABLE'].keys(): + # only ports must be chosen + len(out['ACL_TABLE'][k]) == 1 + return + + def test_break_out(self): + # prepare default config + self.writeJson(portBreakOutConfigDbJson, \ + config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE) + # prepare config dj json to start with + curConfig = dict(configDbJson) + #Ethernet8: start from 4x25G-->2x50G with -f -l + self.dpb_port8_4x25G_2x50G_f_l(curConfig) + #Ethernet8: move from 2x50G-->1x100G without force, list deps + self.dpb_port8_2x50G_1x100G(curConfig) + # Ethernet8: move from 2x50G-->1x100G with force, where deps exists + self.dpb_port8_2x50G_1x100G_f(curConfig) + # Ethernet8: move from 1x100G-->4x25G without force, no deps + self.dpb_port8_1x100G_4x25G(curConfig) + # Ethernet8: move from 4x25G-->1x100G with force, no deps + self.dpb_port8_4x25G_1x100G_f(curConfig) + # Ethernet8: move from 1x100G-->1x50G(2)+2x25G(2) with -f -l, + self.dpb_port8_1x100G_1x50G_2x25G_f_l(curConfig) + # Ethernet4: breakout from 4x25G to 2x50G with -f -l + self.dpb_port4_4x25G_2x50G_f_l(curConfig) + return + + def tearDown(self): + try: + os.remove(config_mgmt.CONFIG_DB_JSON_FILE) + os.remove(config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE) + except Exception as e: + pass + return + + ########### HELPER FUNCS ##################################### + def writeJson(self, d, file): + with open(file, 'w') as f: + dump(d, f, indent=4) + return + + def config_mgmt_dpb(self, curConfig): + ''' + config_mgmt.ConfigMgmtDPB class instance with mocked functions. Not using + pytest fixture, because it is used in non test funcs. + + Parameter: + curConfig (dict): Config to start with. + + Return: + cmdpb (ConfigMgmtDPB): Class instance of ConfigMgmtDPB. + ''' + # create object + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) + # mock funcs + cmdpb.writeConfigDB = MagicMock(return_value=True) + cmdpb._verifyAsicDB = MagicMock(return_value=True) + import mock_tables.dbconnector + return cmdpb + + def generate_args(self, portIdx, laneIdx, curMode, newMode): + ''' + Generate port to deleted, added and {lanes, speed} setting based on + current and new mode. + Example: + For generate_args(8, 73, '4x25G', '2x50G'): + output: + ( + ['Ethernet8', 'Ethernet9', 'Ethernet10', 'Ethernet11'], + ['Ethernet8', 'Ethernet10'], + {'Ethernet8': {'lanes': '73,74', 'speed': '50000'}, + 'Ethernet10': {'lanes': '75,76', 'speed': '50000'}}) + + Parameters: + portIdx (int): Port Index. + laneIdx (int): Lane Index. + curMode (str): current breakout mode of Port. + newMode (str): new breakout mode of Port. + + Return: + dPorts, pJson (tuple)[list, dict] + ''' + # default params + pre = "Ethernet" + laneMap = {"4x25G": [1,1,1,1], "2x50G": [2,2], "1x100G":[4], \ + "1x50G(2)+2x25G(2)":[2,1,1], "2x25G(2)+1x50G(2)":[1,1,2]} + laneSpeed = 25000 + # generate dPorts + l = list(laneMap[curMode]); l.insert(0, 0); id = portIdx; dPorts = list() + for i in l[:-1]: + id = id + i + portName = portName = "{}{}".format(pre, id) + dPorts.append(portName) + # generate aPorts + l = list(laneMap[newMode]); l.insert(0, 0); id = portIdx; aPorts = list() + for i in l[:-1]: + id = id + i + portName = portName = "{}{}".format(pre, id) + aPorts.append(portName) + # generate pJson + l = laneMap[newMode]; pJson = {"PORT": {}}; li = laneIdx; pi = 0 + for i in l: + speed = laneSpeed*i + lanes = [str(li+j) for j in range(i)]; lanes = ','.join(lanes) + pJson['PORT'][aPorts[pi]] = {"speed": str(speed), "lanes": str(lanes)} + li = li+i; pi = pi + 1 + return dPorts, pJson + + def updateConfig(self, conf, uconf): + ''' + update the config to emulate continous breakingout a single port. + + Parameters: + conf (dict): current config in config DB. + uconf (dict): config Diff to be pushed in config DB. + + Return: + void + conf will be updated with uconf, i.e. config diff. + ''' + try: + for it in uconf.keys(): + # if conf has the key + if conf.get(it): + # if marked for deletion + if uconf[it] == None: + del conf[it] + else: + if isinstance(conf[it], list) and isinstance(uconf[it], list): + conf[it] = list(uconf[it]) + elif isinstance(conf[it], dict) and isinstance(uconf[it], dict): + self.updateConfig(conf[it], uconf[it]) + else: + conf[it] = uconf[it] + del uconf[it] + # update new keys in conf + conf.update(uconf) + except Exception as e: + print("update Config failed") + print(e) + raise e + return + + def checkResult(self, cmdpb, delConfig, addConfig): + ''' + Usual result check in many test is: Make sure delConfig and addConfig is + pushed in order to configDb + + Parameters: + cmdpb (ConfigMgmtDPB): Class instance of ConfigMgmtDPB. + delConfig (dict): config Diff to be pushed in config DB while deletion + of ports. + addConfig (dict): config Diff to be pushed in config DB while addition + of ports. + + Return: + void + ''' + calls = [call(delConfig), call(addConfig)] + assert cmdpb.writeConfigDB.call_count == 2 + cmdpb.writeConfigDB.assert_has_calls(calls, any_order=False) + return + + def postUpdateConfig(self, curConfig, delConfig, addConfig): + ''' + After breakout, update the config to emulate continous breakingout a + single port. + + Parameters: + curConfig (dict): current Config in config DB. + delConfig (dict): config Diff to be pushed in config DB while deletion + of ports. + addConfig (dict): config Diff to be pushed in config DB while addition + of ports. + + Return: + void + curConfig will be updated with delConfig and addConfig. + ''' + # update the curConfig with change + self.updateConfig(curConfig, delConfig) + self.updateConfig(curConfig, addConfig) + return + + def dpb_port8_1x100G_1x50G_2x25G_f_l(self, curConfig): + ''' + Breakout Port 8 1x100G->1x50G_2x25G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='1x100G', newMode='1x50G(2)+2x25G(2)') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, + force=True, loadDefConfig=True) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10'] + }, + u'NO-NSW-PACL-TEST': { + u'ports': ['Ethernet11'] + } + }, + u'INTERFACE': { + u'Ethernet11|2a04:1111:40:a709::1/126': { + u'scope': u'global', + u'family': u'IPv6' + }, + u'Ethernet11': {} + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': { + u'tagging_mode': u'untagged' + }, + u'Vlan100|Ethernet11': { + u'tagging_mode': u'untagged' + } + }, + u'PORT': { + 'Ethernet8': { + 'speed': '50000', + 'lanes': '73,74' + }, + 'Ethernet10': { + 'speed': '25000', + 'lanes': '75' + }, + 'Ethernet11': { + 'speed': '25000', + 'lanes': '76' + } + } + } + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + return + + def dpb_port8_4x25G_1x100G_f(self, curConfig): + ''' + Breakout Port 8 4x25G->1x100G with -f + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='4x25G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None, + u'Ethernet9': None, + u'Ethernet10': None, + u'Ethernet11': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + return + + def dpb_port8_1x100G_4x25G(self, curConfig): + ''' + Breakout Port 8 1x100G->4x25G + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='1x100G', newMode='4x25G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + return + + def dpb_port8_2x50G_1x100G_f(self, curConfig): + ''' + Breakout Port 8 2x50G->1x100G with -f + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='2x50G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, + force=True, loadDefConfig=False) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4'] + } + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': None + }, + u'PORT': { + u'Ethernet8': None, + u'Ethernet10': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + + def dpb_port8_2x50G_1x100G(self, curConfig): + ''' + Breakout Port 8 2x50G->1x100G + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='2x50G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result + assert ret == False and len(deps) == 3 + assert cmdpb.writeConfigDB.call_count == 0 + return + + def dpb_port8_4x25G_2x50G_f_l(self, curConfig): + ''' + Breakout Port 8 4x25G->2x50G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='4x25G', newMode='2x50G') + cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, \ + loadDefConfig=True) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4'] + }, + u'NO-NSW-PACL-TEST': { + u'ports': None + } + }, + u'INTERFACE': None, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': None, + u'Vlan100|Ethernet11': None + }, + u'PORT': { + u'Ethernet8': None, + u'Ethernet9': None, + u'Ethernet10': None, + u'Ethernet11': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10'] + } + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': { + u'tagging_mode': u'untagged' + } + }, + u'PORT': { + 'Ethernet8': { + 'speed': '50000', + 'lanes': '73,74' + }, + 'Ethernet10': { + 'speed': '50000', + 'lanes': '75,76' + } + } + } + assert cmdpb.writeConfigDB.call_count == 2 + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + return + + def dpb_port4_4x25G_2x50G_f_l(self, curConfig): + ''' + Breakout Port 4 4x25G->2x50G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) + # create ARGS + dPorts, pJson = self.generate_args(portIdx=4, laneIdx=69, \ + curMode='4x25G', newMode='2x50G') + cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, \ + loadDefConfig=True) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10'] + } + }, + u'PORT': { + u'Ethernet4': None, + u'Ethernet5': None, + u'Ethernet6': None, + u'Ethernet7': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10', 'Ethernet4'] + } + }, + u'PORT': { + 'Ethernet4': { + 'speed': '50000', + 'lanes': '69,70' + }, + 'Ethernet6': { + 'speed': '50000', + 'lanes': '71,72' + } + } + } + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + return + +###########GLOBAL Configs##################################### +configDbJson = { + "ACL_TABLE": { + "NO-NSW-PACL-TEST": { + "policy_desc": "NO-NSW-PACL-TEST", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet9", + "Ethernet11", + ] + }, + "NO-NSW-PACL-V4": { + "policy_desc": "NO-NSW-PACL-V4", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet0", + "Ethernet4", + "Ethernet8", + "Ethernet10" + ] + } + }, + "VLAN": { + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "10.186.72.116" + ] + }, + }, + "VLAN_MEMBER": { + "Vlan100|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet2": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet11": { + "tagging_mode": "untagged" + }, + }, + "INTERFACE": { + "Ethernet10": {}, + "Ethernet10|2a04:0000:40:a709::1/126": { + "scope": "global", + "family": "IPv6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet1": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet2": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet3": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet4": { + "alias": "Eth2/1", + "lanes": "69", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet5": { + "alias": "Eth2/2", + "lanes": "70", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet6": { + "alias": "Eth2/3", + "lanes": "71", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet7": { + "alias": "Eth2/4", + "lanes": "72", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet8": { + "alias": "Eth3/1", + "lanes": "73", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet9": { + "alias": "Eth3/2", + "lanes": "74", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet10": { + "alias": "Eth3/3", + "lanes": "75", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet11": { + "alias": "Eth3/4", + "lanes": "76", + "description": "", + "speed": "25000", + "admin_status": "up" + } + } +} + +portBreakOutConfigDbJson = { + "ACL_TABLE": { + "NO-NSW-PACL-TEST": { + "ports": [ + "Ethernet9", + "Ethernet11", + ] + }, + "NO-NSW-PACL-V4": { + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0", + "Ethernet4", + "Ethernet8", + "Ethernet10" + ] + } + }, + "VLAN": { + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "10.186.72.116" + ] + } + }, + "VLAN_MEMBER": { + "Vlan100|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet11": { + "tagging_mode": "untagged" + } + }, + "INTERFACE": { + "Ethernet11": {}, + "Ethernet11|2a04:1111:40:a709::1/126": { + "scope": "global", + "family": "IPv6" + } + } +} diff --git a/sonic-utilities-tests/mock_tables/counters_db.json b/sonic-utilities-tests/mock_tables/counters_db.json index 2476837d71..2b2b600280 100644 --- a/sonic-utilities-tests/mock_tables/counters_db.json +++ b/sonic-utilities-tests/mock_tables/counters_db.json @@ -145,6 +145,12 @@ "Ethernet4": "oid:0x1000000000004", "Ethernet8": "oid:0x1000000000006" }, + "COUNTERS_LAG_NAME_MAP": { + "PortChannel0001": "oid:0x60000000005a1", + "PortChannel0002": "oid:0x60000000005a2", + "PortChannel0003": "oid:0x600000000063c", + "PortChannel0004": "oid:0x600000000063d" + }, "COUNTERS_DEBUG_NAME_PORT_STAT_MAP": { "DEBUG_0": "SAI_PORT_STAT_IN_DROP_REASON_RANGE_BASE", "DEBUG_2": "SAI_PORT_STAT_OUT_CONFIGURED_DROP_REASONS_1_DROPPED_PKTS"