[201911] Minigraph parser changes to select mmu profiles based on Son…

…icQosProfile attribute (#10927)

Why I did it
There is a need to select different mmu profiles based on deployment type

How I did it
There will be separate subfolders (RDMA-CENTRIC, TCP-CENTRIC, BALANCED) in each hwsku folder which contains deployment specific mmu and qos settings. SonicQosProfile attribute in the minigraph will be used to determine which settings to use. If that attribute is not present, the default settings that exist in the hwsku folder will be used

Signed-off-by: Neetha John <[email protected]>

src/sonic-config-engine/minigraph.py

@@ -610,6 +610,7 @@ def parse_meta(meta, hname):
     region = None
     cloudtype = None
     downstream_subrole = None
+    qos_profile = None
     device_metas = meta.find(str(QName(ns, "Devices")))
     for device in device_metas.findall(str(QName(ns1, "DeviceMetadata"))):
         if device.find(str(QName(ns1, "Name"))).text.lower() == hname.lower():
@@ -638,7 +639,9 @@ def parse_meta(meta, hname):
                     cloudtype = value
                 elif name == "DownStreamSubRole":
                     downstream_subrole = value
-    return syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, downstream_subrole
+                elif name == "SonicQosProfile":
+                    qos_profile = value
+    return syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, downstream_subrole, qos_profile
 
 
 def parse_linkmeta(meta, hname):
@@ -857,6 +860,31 @@ def enable_internal_bgp_session(bgp_sessions, filename, asic_name):
             (local_sub_role == BACKEND_ASIC_SUB_ROLE and peer_sub_role == FRONTEND_ASIC_SUB_ROLE)):
             bgp_sessions[peer_ip].update({'admin_status': 'up'})
 
+def select_mmu_profiles(profile, platform, hwsku):
+    """
+        Select MMU files based on the device metadata attribute - SonicQosProfile
+        if no QosProfile exists in the minigraph, then no action is needed.
+        if a profile exists in the minigraph,
+            - create a dir path to search 1 level down from the base path.
+            - if no such dir path exists, no action is needed.
+            - if a dir path exists, check for the presence of each file from
+              the copy list in the dir path and copy it over to the base path.
+    """
+    if not profile:
+        return
+
+    files_to_copy = ['pg_profile_lookup.ini', 'qos.json.j2', 'buffers_defaults_t0.j2', 'buffers_defaults_t1.j2']
+
+    path = os.path.join('/usr/share/sonic/device', platform, hwsku)
+
+    dir_path = os.path.join(path, profile)
+    if os.path.exists(dir_path):
+        for file_item in files_to_copy:
+            file_in_dir = os.path.join(dir_path, file_item)
+            if os.path.isfile(file_in_dir):
+                base_file = os.path.join(path, file_item)
+                exec_cmd("sudo cp {} {}".format(file_in_dir, base_file))
+
 ###############################################################################
 #
 # Main functions
@@ -912,6 +940,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
     host_lo_intfs = None
     is_storage_device = False
     local_devices = []
+    qos_profile = None
 
     # hostname is the asic_name, get the asic_id from the asic_name
     if asic_name is not None:
@@ -948,7 +977,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
             elif child.tag == str(QName(ns, "UngDec")):
                 (u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname)
             elif child.tag == str(QName(ns, "MetadataDeclaration")):
-                (syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, downstream_subrole) = parse_meta(child, hostname)
+                (syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, downstream_subrole, qos_profile) = parse_meta(child, hostname)
             elif child.tag == str(QName(ns, "LinkMetadataDeclaration")):
                 linkmetas = parse_linkmeta(child, hostname)
             elif child.tag == str(QName(ns, "DeviceInfos")):
@@ -968,6 +997,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
             elif child.tag == str(QName(ns, "DeviceInfos")):
                 (port_speeds_default, port_descriptions) = parse_deviceinfo(child, hwsku)
 
+    select_mmu_profiles(qos_profile, platform, hwsku)
     # set the host device type in asic metadata also
     device_type = [devices[key]['type'] for key in devices if key.lower() == hostname.lower()][0]
     if asic_name is None:

src/sonic-config-engine/tests/sample-dell-6100-t0-minigraph.xml

@@ -731,6 +731,11 @@
             <a:Reference i:nil="true"/>
             <a:Value>True</a:Value>
           </a:DeviceProperty>
+          <a:DeviceProperty>
+            <a:Name>SonicQosProfile</a:Name>
+            <a:Reference i:nil="true"/>
+            <a:Value>RDMA-CENTRIC</a:Value>
+          </a:DeviceProperty>
         </a:Properties>
         <a:Key>ARISTA01T1:Ethernet1;s6100-dev-1:fortyGigE1/1/1</a:Key>
       </a:LinkMetadata>

src/sonic-config-engine/tests/test_j2files.py

@@ -31,6 +31,30 @@ def run_script(self, argument):
     def run_diff(self, file1, file2):
         return subprocess.check_output('diff -u {} {} || true'.format(file1, file2), shell=True)
 
+    def create_machine_conf(self, platform, vendor):
+        file_exist = True
+        dir_exist = True
+        mode = {'arista': 'aboot',
+                'dell': 'onie'
+               }
+        echo_cmd = "echo '{}_platform={}' | sudo tee -a /host/machine.conf > /dev/null".format(mode[vendor], platform)
+        if not os.path.exists('/host/machine.conf'):
+            file_exist = False
+            if not os.path.isdir('/host'):
+                dir_exist = False
+                os.system('sudo mkdir /host')
+            os.system('sudo touch /host/machine.conf')
+            os.system(echo_cmd)
+
+        return file_exist, dir_exist
+
+    def remove_machine_conf(self, file_exist, dir_exist):
+        if not file_exist:
+            os.system('sudo rm -f /host/machine.conf')
+
+        if not dir_exist:
+            os.system('sudo rmdir /host')
+
     def test_interfaces(self):
         interfaces_template = os.path.join(self.test_dir, '..', '..', '..', 'files', 'image_config', 'interfaces', 'interfaces.j2')
         argument = '-m ' + self.t0_minigraph + ' -a \'{\"hwaddr\":\"e4:1d:2d:a5:f3:ad\"}\' -t ' + interfaces_template + ' > ' + self.output_file
@@ -134,44 +158,36 @@ def test_l2switch_template(self):
         self.assertTrue(json.dumps(sample_output_json, sort_keys=True) == json.dumps(output_json, sort_keys=True))
 
     def test_qos_arista7050_render_template(self):
-        arista_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'arista', 'x86_64-arista_7050_qx32s', 'Arista-7050-QX-32S')
-        qos_file = os.path.join(arista_dir_path, 'qos.json.j2')
-        port_config_ini_file = os.path.join(arista_dir_path, 'port_config.ini')
-
-        # copy qos_config.j2 to the Arista 7050 directory to have all templates in one directory
-        qos_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'qos_config.j2')
-        shutil.copy2(qos_config_file, arista_dir_path)
-
-        argument = '-m ' + self.arista7050_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file
-        self.run_script(argument)
-
-        # cleanup
-        qos_config_file_new = os.path.join(arista_dir_path, 'qos_config.j2')
-        os.remove(qos_config_file_new)
-
-        sample_output_file = os.path.join(self.test_dir, 'sample_output', 'qos-arista7050.json')
-        assert filecmp.cmp(sample_output_file, self.output_file)
+        self._test_qos_render_template('arista', 'x86_64-arista_7050_qx32s', 'Arista-7050-QX-32S', 'sample-arista-7050-t0-minigraph.xml', 'qos-arista7050.json')
 
     def test_qos_dell6100_render_template(self):
-        dell_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100')
-        qos_file = os.path.join(dell_dir_path, 'qos.json.j2')
-        port_config_ini_file = os.path.join(dell_dir_path, 'port_config.ini')
+        self._test_qos_render_template('dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100', 'sample-dell-6100-t0-minigraph.xml', 'qos-dell6100.json')
 
-        # copy qos_config.j2 to the Dell S6100 directory to have all templates in one directory
+    def _test_qos_render_template(self, vendor, platform, sku, minigraph, expected):
+        file_exist, dir_exist = self.create_machine_conf(platform, vendor)
+        dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', vendor, platform, sku)
+        qos_file = os.path.join(dir_path, 'qos.json.j2')
+        port_config_ini_file = os.path.join(dir_path, 'port_config.ini')
+
+        # copy qos_config.j2 to the SKU directory to have all templates in one directory
         qos_config_file = os.path.join(self.test_dir, '..', '..', '..', 'files', 'build_templates', 'qos_config.j2')
-        shutil.copy2(qos_config_file, dell_dir_path)
+        shutil.copy2(qos_config_file, dir_path)
 
-        argument = '-m ' + self.dell6100_t0_minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file
+        minigraph = os.path.join(self.test_dir, minigraph)
+        argument = '-m ' + minigraph + ' -p ' + port_config_ini_file + ' -t ' + qos_file + ' > ' + self.output_file
         self.run_script(argument)
 
         # cleanup
-        qos_config_file_new = os.path.join(dell_dir_path, 'qos_config.j2')
+        qos_config_file_new = os.path.join(dir_path, 'qos_config.j2')
         os.remove(qos_config_file_new)
 
-        sample_output_file = os.path.join(self.test_dir, 'sample_output', 'qos-dell6100.json')
-        assert filecmp.cmp(sample_output_file, self.output_file)
+        self.remove_machine_conf(file_exist, dir_exist)
+
+        sample_output_file = os.path.join(self.test_dir, 'sample_output', expected)
+        assert filecmp.cmp(sample_output_file, self.output_file), self.run_diff(sample_output_file, self.output_file)
 
     def test_buffers_dell6100_render_template(self):
+        file_exist, dir_exist = self.create_machine_conf('x86_64-dell_s6100_c2538-r0', 'dell')
         dell_dir_path = os.path.join(self.test_dir, '..', '..', '..', 'device', 'dell', 'x86_64-dell_s6100_c2538-r0', 'Force10-S6100')
         buffers_file = os.path.join(dell_dir_path, 'buffers.json.j2')
         port_config_ini_file = os.path.join(dell_dir_path, 'port_config.ini')
@@ -187,6 +203,8 @@ def test_buffers_dell6100_render_template(self):
         buffers_config_file_new = os.path.join(dell_dir_path, 'buffers_config.j2')
         os.remove(buffers_config_file_new)
 
+        self.remove_machine_conf(file_exist, dir_exist)
+
         sample_output_file = os.path.join(self.test_dir, 'sample_output', 'buffers-dell6100.json')
         assert filecmp.cmp(sample_output_file, self.output_file)