[sonic-yang-mgmt] Build PY3 & PY2 packages

files/build_templates/sonic_debian_extension.j2

@@ -154,15 +154,23 @@ sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY2_WHEEL_NAME
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb
 SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}})
 sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
-# Install sonic-yang-mgmt Python package
-SONIC_YANG_MGMT_PY_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py_wheel_path}})
-sudo cp {{sonic_yang_mgmt_py_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install $SONIC_YANG_MGMT_PY_WHEEL_NAME
-sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY_WHEEL_NAME
+
+# Install sonic-yang-mgmt Python2 package
+SONIC_YANG_MGMT_PY2_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py2_wheel_path}})
+sudo cp {{sonic_yang_mgmt_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install $SONIC_YANG_MGMT_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY2_WHEEL_NAME
+
+# Install sonic-yang-mgmt Python3 package
+SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}})
+sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
 
 # Install sonic-platform-common Python 2 package
 PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})

platform/vs/docker-sonic-vs.mk

@@ -22,7 +22,7 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
                                     $(SONIC_PY_COMMON_PY2) \
                                     $(SONIC_PY_COMMON_PY3) \
                                     $(SONIC_YANG_MODELS_PY3) \
-                                    $(SONIC_YANG_MGMT_PY) \
+                                    $(SONIC_YANG_MGMT_PY2) \
                                     $(SONIC_UTILITIES_PY2) \
                                     $(SONIC_HOST_SERVICES_PY3)
 

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -83,11 +83,6 @@ RUN pip2 install urllib3
 RUN pip2 install requests
 RUN pip2 install crontab
 
-# Install dependencies for Dynamic Port Breakout
-RUN pip2 install xmltodict==0.12.0
-RUN pip2 install jsondiff==1.2.0
-RUN pip2 install ijson==2.6.1
-
 {% if docker_sonic_vs_debs.strip() -%}
 # Copy locally-built Debian package dependencies
 {%- for deb in docker_sonic_vs_debs.split(' ') %}

rules/sonic-utilities.mk

@@ -14,7 +14,7 @@ $(SONIC_UTILITIES_PY2)_DEPENDS += $(SONIC_PY_COMMON_PY2) \
                                   $(SONIC_PY_COMMON_PY3) \
                                   $(SWSSSDK_PY2) \
                                   $(SONIC_CONFIG_ENGINE_PY2) \
-                                  $(SONIC_YANG_MGMT_PY) \
+                                  $(SONIC_YANG_MGMT_PY2) \
                                   $(SONIC_YANG_MODELS_PY3)
 $(SONIC_UTILITIES_PY2)_DEBS_DEPENDS = $(LIBYANG) \
                                       $(LIBYANG_CPP) \

rules/sonic-yang-mgmt-py2.dep

@@ -1,10 +1,10 @@
 
-SPATH       := $($(SONIC_YANG_MGMT_PY)_SRC_PATH)
+SPATH       := $($(SONIC_YANG_MGMT_PY2)_SRC_PATH)
 DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-yang-mgmt-py2.mk rules/sonic-yang-mgmt-py2.dep   
 DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
 DEP_FILES   += $(shell git ls-files $(SPATH))
 
-$(SONIC_YANG_MGMT_PY)_CACHE_MODE  := GIT_CONTENT_SHA 
-$(SONIC_YANG_MGMT_PY)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
-$(SONIC_YANG_MGMT_PY)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_YANG_MGMT_PY2)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_YANG_MGMT_PY2)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_YANG_MGMT_PY2)_DEP_FILES   := $(DEP_FILES)
 

rules/sonic-yang-mgmt-py2.mk

@@ -1,12 +1,12 @@
 # sonic-yang-mgmt python2 wheel
 
-SONIC_YANG_MGMT_PY = sonic_yang_mgmt-1.0-py2-none-any.whl
-$(SONIC_YANG_MGMT_PY)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
-$(SONIC_YANG_MGMT_PY)_PYTHON_VERSION = 2
-$(SONIC_YANG_MGMT_PY)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY2) \
+SONIC_YANG_MGMT_PY2 = sonic_yang_mgmt-1.0-py2-none-any.whl
+$(SONIC_YANG_MGMT_PY2)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
+$(SONIC_YANG_MGMT_PY2)_PYTHON_VERSION = 2
+$(SONIC_YANG_MGMT_PY2)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY2) \
                                      $(LIBYANG_PY3)
-$(SONIC_YANG_MGMT_PY)_DEPENDS =  $(SONIC_YANG_MODELS_PY3)
-$(SONIC_YANG_MGMT_PY)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
+$(SONIC_YANG_MGMT_PY2)_DEPENDS =  $(SONIC_YANG_MODELS_PY3)
+$(SONIC_YANG_MGMT_PY2)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
                                  $(LIBYANG_CPP) $(LIBYANG_PY2)
 
-SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY)
+SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY2)

rules/sonic-yang-mgmt-py3.dep

@@ -0,0 +1,10 @@
+
+SPATH       := $($(SONIC_YANG_MGMT_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-yang-mgmt-py3.mk rules/sonic-yang-mgmt-py3.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(SPATH))
+
+$(SONIC_YANG_MGMT_PY3)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_YANG_MGMT_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_YANG_MGMT_PY3)_DEP_FILES   := $(DEP_FILES)
+

rules/sonic-yang-mgmt-py3.mk

@@ -0,0 +1,11 @@
+# sonic-yang-mgmt python3 wheel
+
+SONIC_YANG_MGMT_PY3 = sonic_yang_mgmt-1.0-py3-none-any.whl
+$(SONIC_YANG_MGMT_PY3)_SRC_PATH = $(SRC_PATH)/sonic-yang-mgmt
+$(SONIC_YANG_MGMT_PY3)_PYTHON_VERSION = 3
+$(SONIC_YANG_MGMT_PY3)_DEBS_DEPENDS = $(LIBYANG) $(LIBYANG_CPP) $(LIBYANG_PY3)
+$(SONIC_YANG_MGMT_PY3)_DEPENDS =  $(SONIC_YANG_MODELS_PY3)
+$(SONIC_YANG_MGMT_PY3)_RDEPENDS = $(SONIC_YANG_MODELS_PY3) $(LIBYANG) \
+                                 $(LIBYANG_CPP) $(LIBYANG_PY3)
+
+SONIC_PYTHON_WHEELS += $(SONIC_YANG_MGMT_PY3)

slave.mk

@@ -821,7 +821,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
-        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2)) \
+        $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \
         $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))
 	$(HEADER)
@@ -864,7 +865,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))"
 	export install_debug_image="$(INSTALL_DEBUG_TOOLS)"
 	export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))"
-	export sonic_yang_mgmt_py_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))"
+	export sonic_yang_mgmt_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))"
+	export sonic_yang_mgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3))"
 	export multi_instance="false"
 	export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
 	export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))"

sonic-slave-buster/Dockerfile.j2

@@ -390,11 +390,7 @@ RUN pip2 install pexpect==4.6.0
 # For sonic-swss-common testing
 RUN pip2 install Pympler==0.8
 
-# For sonic_yang_mgmt build
-RUN pip2 install ijson==2.6.1
-RUN pip3 install ijson==2.6.1
-RUN pip2 install jsondiff==1.2.0
-RUN pip2 install xmltodict==0.12.0
+# For sonic_yang_model build
 RUN pip2 install pyang==2.1.1
 
 # For mgmt-framework build

sonic-slave-stretch/Dockerfile.j2

@@ -339,7 +339,7 @@ RUN pip2 install \
          ctypesgen==0.r125 \
          crc16
 
-# Note: Stick with Jinja2 2.x branch as the 3.x dropped support for Python 2.7 
+# Note: Stick with Jinja2 2.x branch as the 3.x dropped support for Python 2.7
 RUN pip2 install --force-reinstall --upgrade "Jinja2<3.0.0"
 
 # For sonic config engine testing
@@ -379,11 +379,7 @@ RUN pip2 install setuptools==40.8.0
 # For sonic-swss-common testing
 RUN pip2 install Pympler==0.8
 
-# For sonic_yang_mgmt build
-RUN pip2 install ijson==2.6.1
-RUN pip3 install ijson==2.6.1
-RUN pip2 install jsondiff==1.2.0
-RUN pip2 install xmltodict==0.12.0
+# For sonic_yang_model build
 RUN pip2 install pyang==2.1.1
 
 # For mgmt-framework build

src/sonic-yang-mgmt/setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

src/sonic-yang-mgmt/setup.py

@@ -4,37 +4,12 @@
 """The setup script."""
 
 from setuptools import setup, find_packages
-from setuptools.command.build_py import build_py
-from os import system
-from sys import exit
-import pytest
-
-setup_requirements = ['pytest-runner']
-
-test_requirements = ['pytest>=3']
 
 # read me
 with open('README.rst') as readme_file:
     readme = readme_file.read()
 
-# class for prerequisites to build this package
-class pkgBuild(build_py):
-    """Custom Build PLY"""
-
-    def run (self):
-        # run pytest for libyang python APIs
-        self.pytest_args = []
-        errno = pytest.main(self.pytest_args)
-        if (errno):
-            exit(errno)
-
-        # Continue usual build steps
-        build_py.run(self)
-
 setup(
-    cmdclass={
-        'build_py': pkgBuild,
-    },
     author="lnos-coders",
     author_email='[email protected]',
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
@@ -52,15 +27,26 @@ def run (self):
         'Programming Language :: Python :: 3.8',
     ],
     description="Package contains Python Library for YANG for sonic.",
-    tests_require = test_requirements,
     license="GNU General Public License v3",
     long_description=readme + '\n\n',
+    install_requires = [
+        'xmltodict==0.12.0',
+        'ijson==2.6.1'
+    ],
+    tests_require = [
+        'pytest>3',
+        'xmltodict==0.12.0',
+        'ijson==2.6.1'
+    ],
+    setup_requires = [
+        'pytest-runner',
+        'wheel'
+    ],
     include_package_data=True,
-    keywords='sonic_yang_mgmt',
-    name='sonic_yang_mgmt',
+    keywords='sonic-yang-mgmt',
+    name='sonic-yang-mgmt',
     py_modules=['sonic_yang', 'sonic_yang_ext'],
     packages=find_packages(),
-    setup_requires=setup_requirements,
     version='1.0',
     zip_safe=False,
 )

src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -129,7 +129,8 @@ def _getModuleTLCcontainer(self, table):
     """
     def _cropConfigDB(self, croppedFile=None):
 
-        for table in self.jIn.keys():
+        tables = list(self.jIn.keys())
+        for table in tables:
             if table not in self.confDbYangMap:
                 # store in tablesWithOutYang
                 self.tablesWithOutYang[table] = self.jIn[table]
@@ -138,7 +139,7 @@ def _cropConfigDB(self, croppedFile=None):
         if len(self.tablesWithOutYang):
             print("Note: Below table(s) have no YANG models:")
             for table in self.tablesWithOutYang.keys():
-                print(unicode(table), end=", ")
+                print(str(table), end=", ")
             print()
 
         if croppedFile:
@@ -274,13 +275,14 @@ def _xlateList(self, model, yang, config, table):
         # fetch regex from YANG models.
         keyRegEx = model['ext:key-regex-configdb-to-yang']['@value']
         # seperator `|` has special meaning in regex, so change it appropriately.
-        keyRegEx = re.sub('\|', '\\|', keyRegEx)
+        keyRegEx = re.sub(r'\|', r'\\|', keyRegEx)
         # get keys from YANG model list itself
         listKeys = model['key']['@value']
         self.sysLog(msg="xlateList regex:{} keyList:{}".\
             format(keyRegEx, listKeys))
 
-        for pkey in config.keys():
+        primaryKeys = list(config.keys())
+        for pkey in primaryKeys:
             try:
                 vKey = None
                 self.sysLog(syslog.LOG_DEBUG, "xlateList Extract pkey:{}".\

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -64,8 +64,8 @@ def setup_class(self):
     def load_yang_model_file(self, yang_s, yang_dir, yang_file, module_name):
         yfile = yang_dir + yang_file
         try:
-	    yang_s._load_schema_module(str(yfile))
-	except Exception as e:
+            yang_s._load_schema_module(str(yfile))
+        except Exception as e:
             print(e)
             raise
 
@@ -106,7 +106,7 @@ def test_load_yang_model_data(self, data, yang_s):
         data_files = []
         data_files.append(data_file)
         data_files.append(data_merge_file)
-	print(yang_files)
+        print(yang_files)
         yang_s._load_data_model(yang_dir, yang_files, data_files)
 
         #validate the data tree from data_merge_file is loaded

src/sonic-yang-models/setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

src/sonic-yang-models/setup.py

@@ -4,51 +4,12 @@
 """The setup script."""
 
 from setuptools import setup, find_packages
-from setuptools.command.build_py import build_py
-from os import system
-from sys import exit
-
-setup_requirements = ['pytest-runner']
-
-test_requirements = ['pytest>=3']
 
 # read me
 with open('README.rst') as readme_file:
     readme = readme_file.read()
 
-# class for prerequisites to build this package
-class pkgBuild(build_py):
-    """Custom Build PLY"""
-
-    def run (self):
-        # json file for YANG model test cases.
-        test_yangJson_file = './tests/yang_model_tests/yangTest.json'
-        # YANG models are in below dir
-        yang_model_dir = './yang-models/'
-        # yang model tester python module
-        yang_test_py = './tests/yang_model_tests/yangModelTesting.py'
-        #  run tests for yang models
-        test_yang_cmd = "python {} -f {} -y {}".format(yang_test_py, test_yangJson_file, yang_model_dir)
-        if (system(test_yang_cmd)):
-            print("YANG Tests failed\n")
-            exit(1)
-        else:
-            print("YANG Tests passed\n")
-
-        # Generate YANG Tree
-        pyang_tree_cmd = "pyang -f tree ./yang-models/*.yang > ./yang-models/sonic_yang_tree"
-        if (system(pyang_tree_cmd)):
-            print("Failed: {}".format(pyang_tree_cmd))
-        else:
-            print("Passed: {}".format(pyang_tree_cmd))
-
-        # Continue usual build steps
-        build_py.run(self)
-
 setup(
-    cmdclass={
-        'build_py': pkgBuild,
-    },
     author="lnos-coders",
     author_email='[email protected]',
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
@@ -66,15 +27,23 @@ def run (self):
         'Programming Language :: Python :: 3.8',
     ],
     description="Package contains YANG models for sonic.",
-    tests_require = test_requirements,
     license="GNU General Public License v3",
     long_description=readme + '\n\n',
+    install_requires = [
+    ],
+    tests_require = [
+        'pytest',
+        'ijson==2.6.1'
+    ],
+    setup_requires = [
+        'pytest-runner',
+        'wheel'
+    ],
     include_package_data=True,
-    keywords='sonic_yang_models',
-    name='sonic_yang_models',
+    keywords='sonic-yang-models',
+    name='sonic-yang-models',
     py_modules=[],
     packages=find_packages(),
-    setup_requires=setup_requirements,
     version='1.0',
     data_files=[
         ('yang-models', ['./yang-models/sonic-types.yang',