From 83108d9c9a27b69894f3f9a8fa3cf05fbb3b2064 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 30 Sep 2021 12:53:34 -0700 Subject: [PATCH] [YANG MGMT]: Support Grouping translation in YANG Models. (#8318) Changes: -- pre Process Grouping section from all yang models, so it can be used from any yang model. -- add jsondiff in setup.py, it is useful for test debugging in case of failures. -- use 'stypes' instead of head. -- pass config DB table name in _createLeafDict(). -- added test config for grouping. -- white spaces changes. Note: Changes are done in the way that we can add support for other Generic YANG statement easily for translation. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- src/sonic-yang-mgmt/setup.py | 6 +- src/sonic-yang-mgmt/sonic_yang.py | 3 + src/sonic-yang-mgmt/sonic_yang_ext.py | 173 ++++++++++++++++-- .../libyang-python-tests/test_sonic_yang.py | 6 +- .../tests/files/sample_config_db.json | 10 +- 5 files changed, 175 insertions(+), 23 deletions(-) diff --git a/src/sonic-yang-mgmt/setup.py b/src/sonic-yang-mgmt/setup.py index e1b037cdcdfa..e97343ac0e0a 100644 --- a/src/sonic-yang-mgmt/setup.py +++ b/src/sonic-yang-mgmt/setup.py @@ -26,12 +26,14 @@ long_description=readme + '\n\n', install_requires = [ 'xmltodict==0.12.0', - 'ijson==2.6.1' + 'ijson==2.6.1', + 'jsondiff>=1.2.0', ], tests_require = [ 'pytest>3', 'xmltodict==0.12.0', - 'ijson==2.6.1' + 'ijson==2.6.1', + 'jsondiff>=1.2.0' ], setup_requires = [ 'pytest-runner', diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index ba982f8556e8..0f62eccd93c1 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -36,6 +36,9 @@ def __init__(self, yang_dir, debug=False): self.revXlateJson = dict() # below dict store the input config tables which have no YANG models self.tablesWithOutYang = dict() + # below dict will store preProcessed yang objects, which may be needed by + # all yang modules, such as grouping. + self.preProcessedYang = dict() try: self.ctx = ly.Context(yang_dir) diff --git a/src/sonic-yang-mgmt/sonic_yang_ext.py b/src/sonic-yang-mgmt/sonic_yang_ext.py index 329db9d28b4a..5438c81fb745 100644 --- a/src/sonic-yang-mgmt/sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/sonic_yang_ext.py @@ -44,7 +44,6 @@ def loadYangModel(self): self._loadJsonYangModel() # create a map from config DB table to yang container self._createDBTableToModuleMap() - except Exception as e: self.sysLog(msg="Yang Models Load failed:{}".format(str(e)), \ debug=syslog.LOG_ERR, doPrint=True) @@ -71,6 +70,70 @@ def _loadJsonYangModel(self): return + def _preProcessYangGrouping(self, moduleName, module): + ''' + PreProcess Grouping Section of YANG models, and store it in + self.preProcessedYang['grouping'] as + {'': + {'': + [] + } + } + + Parameters: + moduleName (str): name of yang module. + module (dict): json format of yang module. + + Returns: + void + ''' + try: + # create grouping dict + if self.preProcessedYang.get('grouping') is None: + self.preProcessedYang['grouping'] = dict() + self.preProcessedYang['grouping'][moduleName] = dict() + + # get groupings from yang module + groupings = module['grouping'] + + # if grouping is a dict, make it a list for common processing + if isinstance(groupings, dict): + groupings = [groupings] + + for grouping in groupings: + gName = grouping["@name"] + gLeaf = grouping["leaf"] + self.preProcessedYang['grouping'][moduleName][gName] = gLeaf + + except Exception as e: + self.sysLog(msg="_preProcessYangGrouping failed:{}".format(str(e)), \ + debug=syslog.LOG_ERR, doPrint=True) + raise e + return + + # preProcesss Generic Yang Objects + def _preProcessYang(self, moduleName, module): + ''' + PreProcess Generic Section of YANG models by calling + _preProcessYang methods. + + Parameters: + moduleName (str): name of yang module. + module (dict): json format of yang module. + + Returns: + void + ''' + try: + # preProcesss Grouping + if module.get('grouping') is not None: + self._preProcessYangGrouping(moduleName, module) + except Exception as e: + self.sysLog(msg="_preProcessYang failed:{}".format(str(e)), \ + debug=syslog.LOG_ERR, doPrint=True) + raise e + return + """ Create a map from config DB tables to container in yang model This module name and topLevelContainer are fetched considering YANG models are @@ -82,6 +145,8 @@ def _createDBTableToModuleMap(self): for j in self.yJson: # get module name moduleName = j['module']['@name'] + # preProcesss Generic Yang Objects + self._preProcessYang(moduleName, j['module']) # get top level container topLevelContainer = j['module'].get('container') # if top level container is none, this is common yang files, which may @@ -104,14 +169,16 @@ def _createDBTableToModuleMap(self): self.confDbYangMap[c['@name']] = { "module" : moduleName, "topLevelContainer": topLevelContainer['@name'], - "container": c + "container": c, + "yangModule": j['module'] } # container is a dict else: self.confDbYangMap[container['@name']] = { "module" : moduleName, "topLevelContainer": topLevelContainer['@name'], - "container": container + "container": container, + "yangModule": j['module'] } return @@ -201,13 +268,87 @@ def _fillSteps(leaf): return - """ - create a dict to map each key under primary key with a dict yang model. - This is done to improve performance of mapping from values of TABLEs in - config DB to leaf in YANG LIST. - """ - def _createLeafDict(self, model): + def _findYangModuleFromPrefix(self, prefix, module): + ''' + Find yang module name from prefix used in given yang module. + Parameters: + prefix (str): prefix used in given yang module. + module (dict): json format of yang module. + + Returns: + (str): module name or None + ''' + try: + # get imports + yangImports = module.get("import"); + if yangImports is None: + return None + # make a list + if isinstance(yangImports, dict): + yangImports = [yangImports] + # find module for given prefix + for yImport in yangImports: + if yImport['prefix']['@value'] == prefix: + return yImport['@module'] + except Exception as e: + self.sysLog(msg="_findYangModuleFromPrefix failed:{}".format(str(e)), \ + debug=syslog.LOG_ERR, doPrint=True) + raise e + return None + + def _fillLeafDictUses(self, uses_s, table, leafDict): + ''' + Find the leaf(s) in a grouping which maps to given uses statement, + then fill leafDict with leaf(s) information. + + Parameters: + uses_s (str): uses statement in yang module. + table (str): config DB table, this table is being translated. + leafDict (dict): dict with leaf(s) information for List\Container + corresponding to config DB table. + + Returns: + (void) + ''' + try: + # make a list + if isinstance(uses_s, dict): + uses_s = [uses_s] + # find yang module for current table + table_module = self.confDbYangMap[table]['yangModule'] + # uses Example: "@name": "bgpcmn:sonic-bgp-cmn" + for uses in uses_s: + # Assume ':' means reference to another module + if ':' in uses['@name']: + prefix = uses['@name'].split(':')[0].strip() + uses_module = self._findYangModuleFromPrefix(prefix, table_module) + else: + uses_module = table_module + grouping = uses['@name'].split(':')[-1].strip() + leafs = self.preProcessedYang['grouping'][uses_module][grouping] + self._fillLeafDict(leafs, leafDict) + except Exception as e: + self.sysLog(msg="_fillLeafDictUses failed:{}".format(str(e)), \ + debug=syslog.LOG_ERR, doPrint=True) + raise e + + return + + def _createLeafDict(self, model, table): + ''' + create a dict to map each key under primary key with a leaf in yang model. + This is done to improve performance of mapping from values of TABLEs in + config DB to leaf in YANG LIST. + + Parameters: + module (dict): json format of yang module. + table (str): config DB table, this table is being translated. + + Returns: + leafDict (dict): dict with leaf(s) information for List\Container + corresponding to config DB table. + ''' leafDict = dict() #Iterate over leaf, choices and leaf-list. self._fillLeafDict(model.get('leaf'), leafDict) @@ -223,6 +364,10 @@ def _createLeafDict(self, model): # leaf-lists self._fillLeafDict(model.get('leaf-list'), leafDict, True) + # uses should map to grouping, + if model.get('uses') is not None: + self._fillLeafDictUses(model.get('uses'), table, leafDict) + return leafDict """ @@ -245,7 +390,7 @@ def _yangConvert(val): elif 'leafref' in type: vValue = val #TODO: find type in sonic-head, as of now, all are enumeration - elif 'head:' in type: + elif 'stypes:' in type: vValue = val else: vValue = val @@ -275,7 +420,7 @@ def _xlateList(self, model, yang, config, table, exceptionList): #create a dict to map each key under primary key with a dict yang model. #This is done to improve performance of mapping from values of TABLEs in #config DB to leaf in YANG LIST. - leafDict = self._createLeafDict(model) + leafDict = self._createLeafDict(model, table) # get keys from YANG model list itself listKeys = model['key']['@value'] @@ -380,7 +525,7 @@ def _xlateContainer(self, model, yang, config, table): self._xlateContainerInContainer(modelContainer, yang, configC, table) ## Handle other leaves in container, - leafDict = self._createLeafDict(model) + leafDict = self._createLeafDict(model, table) vKeys = list(configC.keys()) for vKey in vKeys: #vkey must be a leaf\leaf-list\choice in container @@ -494,7 +639,7 @@ def _revXlateList(self, model, yang, config, table): # create a dict to map each key under primary key with a dict yang model. # This is done to improve performance of mapping from values of TABLEs in # config DB to leaf in YANG LIST. - leafDict = self._createLeafDict(model) + leafDict = self._createLeafDict(model, table) # list with name _LIST should be removed, if "_LIST" in model['@name']: @@ -559,7 +704,7 @@ def _revXlateContainer(self, model, yang, config, table): self._revXlateContainerInContainer(modelContainer, yang, config, table) ## Handle other leaves in container, - leafDict = self._createLeafDict(model) + leafDict = self._createLeafDict(model, table) for vKey in yang: #vkey must be a leaf\leaf-list\choice in container if leafDict.get(vKey): diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index e52f214f372c..cdec84c6972f 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -341,10 +341,8 @@ def test_xlate_rev_xlate(self, sonic_yang_data): else: print("Xlate and Rev Xlate failed") # print for better debugging, in case of failure. - print("syc.jIn: {}".format({t:syc.jIn[t].keys() \ - for t in syc.jIn.keys()})) - print("syc.revXlateJson: {}".format({t:syc.revXlateJson[t].keys() \ - for t in syc.revXlateJson.keys()})) + from jsondiff import diff + print(diff(syc.jIn, syc.revXlateJson, syntax='symmetric')) # make it fail assert False == True diff --git a/src/sonic-yang-models/tests/files/sample_config_db.json b/src/sonic-yang-models/tests/files/sample_config_db.json index 18ead92d29de..62bff9c8d36e 100644 --- a/src/sonic-yang-models/tests/files/sample_config_db.json +++ b/src/sonic-yang-models/tests/files/sample_config_db.json @@ -928,7 +928,7 @@ }, "AAA": { "authentication": { - "login": "local" + "login": "local" } }, "TACPLUS": { @@ -1009,6 +1009,10 @@ "rrclient":"0" }, "default|192.168.1.1": { + "local_asn": "65200", + "asn": "65100", + "name": "bgp peer 65100", + "ebgp_multihop_ttl": "3" } }, "BGP_NEIGHBOR_AF": { @@ -1043,7 +1047,7 @@ }, "PREFIX_SET": { "prefix1": { - } + } }, "PREFIX": { "prefix1|1|10.0.0.0/8|8..16": { @@ -1093,5 +1097,5 @@ "Error": "This Table is for testing, This Table does not have YANG models." } } - + }