[warm-reboot][sad] enhance test to cover different port types and ope…

ansible/roles/test/files/ptftests/advanced-reboot.py

@@ -156,6 +156,7 @@ def __init__(self):
         self.check_param('vnet_pkts', None, required=False)
         self.check_param('target_version', '', required=False)
         self.check_param('bgp_v4_v6_time_diff', 40, required=False)
+        self.check_param('logfile_suffix', None, required=False)
         if not self.test_params['preboot_oper'] or self.test_params['preboot_oper'] == 'None':
             self.test_params['preboot_oper'] = None
         if not self.test_params['inboot_oper'] or self.test_params['inboot_oper'] == 'None':
@@ -167,9 +168,14 @@ def __init__(self):
         else:
             self.sad_oper = self.test_params['inboot_oper']
 
-        if self.sad_oper:
-           self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.sad_oper)
-           self.report_file_name = '/tmp/%s-%s.json' % (self.test_params['reboot_type'], self.sad_oper)
+        if self.test_params['logfile_suffix']:
+            self.logfile_suffix = self.test_params['logfile_suffix']
+        else:
+            self.logfile_suffix = self.sad_oper
+
+        if self.logfile_suffix:
+           self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.logfile_suffix)
+           self.report_file_name = '/tmp/%s-%s.json' % (self.test_params['reboot_type'], self.logfile_suffix)
         else:
            self.log_file_name = '/tmp/%s.log' % self.test_params['reboot_type']
            self.report_file_name = '/tmp/%s-report.json' % self.test_params['reboot_type']
@@ -354,7 +360,7 @@ def generate_arp_responder_conf(self, vlan_host_map):
 
     def dump_arp_responder_config(self, dump):
         # save data for arp_replay process
-        filename = "/tmp/from_t1.json" if self.sad_oper is None else "/tmp/from_t1_%s.json" % self.sad_oper
+        filename = "/tmp/from_t1.json" if self.logfile_suffix is None else "/tmp/from_t1_%s.json" % self.logfile_suffix
         with open(filename, "w") as fp:
             json.dump(dump, fp)
 
@@ -1356,7 +1362,7 @@ def sniff_in_background(self, wait = None):
         self.sniffer_started.clear()
 
     def save_sniffed_packets(self):
-        filename = "/tmp/capture_%s.pcap" % self.sad_oper if self.sad_oper is not None else "/tmp/capture.pcap"
+        filename = "/tmp/capture_%s.pcap" % self.logfile_suffix if self.logfile_suffix is not None else "/tmp/capture.pcap"
         if self.packets:
             scapyall.wrpcap(filename, self.packets)
             self.log("Pcap file dumped to %s" % filename)
@@ -1510,7 +1516,7 @@ def examine_flow(self, filename = None):
             self.log("Gaps in forwarding not found.")
         self.log("Total incoming packets captured %d" % received_counter)
         if packets:
-            filename = '/tmp/capture_filtered.pcap' if self.sad_oper is None else "/tmp/capture_filtered_%s.pcap" % self.sad_oper
+            filename = '/tmp/capture_filtered.pcap' if self.logfile_suffix is None else "/tmp/capture_filtered_%s.pcap" % self.logfile_suffix
             scapyall.wrpcap(filename, packets)
             self.log("Filtered pcap dumped to %s" % filename)
 

tests/common/devices/sonic.py

@@ -769,6 +769,7 @@ def shutdown(self, ifname):
             Args:
                 ifname: the interface to shutdown
         """
+        logging.info("Shutting down {}".format(ifname))
         return self.command("sudo config interface shutdown {}".format(ifname))
 
     def shutdown_multiple(self, ifnames):
@@ -788,6 +789,7 @@ def no_shutdown(self, ifname):
             Args:
                 ifname: the interface to bring up
         """
+        logging.info("Starting up {}".format(ifname))
         return self.command("sudo config interface startup {}".format(ifname))
 
     def no_shutdown_multiple(self, ifnames):

tests/common/fixtures/advanced_reboot.py

@@ -1,3 +1,4 @@
+import copy
 import ipaddress
 import itertools
 import json
@@ -8,6 +9,7 @@
 from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice
 from tests.common.platform.ssh_utils import prepare_testbed_ssh_keys as prepareTestbedSshKeys
 from tests.common.reboot import reboot as rebootDut
+from tests.common.helpers.sad_path import SadOperation
 from tests.ptf_runner import ptf_runner
 
 logger = logging.getLogger(__name__)
@@ -199,6 +201,10 @@ def __validateAndBuildSadList(self):
         sadList = [item for item in itertools.chain(prebootList, inbootList)]
 
         for item in sadList:
+            # TODO: Move all sad path logic out of ptf script to pytest.
+            # Once done, we can make a sad_operation fixture.
+            if isinstance(item, SadOperation):
+                continue
             if ':' not in item:
                 continue
             itemCnt = int(item.split(':')[-1])
@@ -212,16 +218,6 @@ def __validateAndBuildSadList(self):
                     'Lag count is greater than or equal to number of VM hosts. '
                     'Current val = {0} Max val = {1}'
                 ).format(itemCnt, self.hostMaxLen)
-            if 'lag_member_down' in item:
-                assert itemCnt <= self.lagMemberCnt, (
-                    'Lag member count is greater than available number of lag members. '
-                    'Current val = {0} Available cnt = {1}'
-                ).format(itemCnt, self.lagMemberCnt)
-            if 'vlan_port_down' in item:
-                assert itemCnt <= self.vlanMaxCnt, (
-                    'Vlan count is greater than or equal to number of Vlan interfaces. '
-                    'Current val = {0} Max val = {1}'
-                ).format(itemCnt, self.vlanMaxCnt)
             if 'routing' in item:
                 assert itemCnt <= self.hostMaxCnt, (
                     'Number of prefixes is greater than allowed max. '
@@ -330,15 +326,6 @@ def __setupTestbed(self):
         '''
         Sets testbed up. It tranfers test data files, ARP responder, and runs script to update IPs and MAC addresses.
         '''
-        testDataFiles = [
-            {'source' : self.mgFacts['minigraph_portchannels'], 'name' : 'portchannel_interfaces'},
-            {'source' : self.mgFacts['minigraph_vlans'],        'name' : 'vlan_interfaces'       },
-            {'source' : self.mgFacts['minigraph_ptf_indices'],  'name' : 'ports'                 },
-            {'source' : self.mgFacts['minigraph_devices'],      'name' : 'peer_dev_info'         },
-            {'source' : self.mgFacts['minigraph_neighbors'],    'name' : 'neigh_port_info'       },
-        ]
-        self.__transferTestDataFiles(testDataFiles, self.ptfhost)
-
         self.__runScript(['remove_ip.sh'], self.ptfhost)
 
         self.__prepareTestbedSshKeys()
@@ -441,11 +428,14 @@ def runRebootTest(self):
         for rebootOper in self.rebootData['sadList']:
             count += 1
             try:
+                self.__setupRebootOper(rebootOper)
                 result = self.__runPtfRunner(rebootOper)
+                self.__verifyRebootOper(rebootOper)
             finally:
                 # always capture the test logs
                 self.__fetchTestLogs(rebootOper)
                 self.__clearArpAndFdbTables()
+                self.__revertRebootOper(rebootOper)
             if not result:
                 return result
             if len(self.rebootData['sadList']) > 1 and count != len(self.rebootData['sadList']):
@@ -463,19 +453,82 @@ def runRebootTestcase(self, prebootList=None, inbootList=None, prebootFiles=None
         self.imageInstall(prebootList, inbootList, prebootFiles)
         return self.runRebootTest()
 
+    def __setupRebootOper(self, rebootOper):
+        testData = {
+            'portchannel_interfaces': copy.deepcopy(self.mgFacts['minigraph_portchannels']),
+            'vlan_interfaces': copy.deepcopy(self.mgFacts['minigraph_vlans']),
+            'ports': copy.deepcopy(self.mgFacts['minigraph_ptf_indices']),
+            'peer_dev_info': copy.deepcopy(self.mgFacts['minigraph_devices']),
+            'neigh_port_info': copy.deepcopy(self.mgFacts['minigraph_neighbors']),
+        }
+
+        if isinstance(rebootOper, SadOperation):
+            logger.info('Running setup handler for reboot operation {}'.format(rebootOper))
+            rebootOper.setup(testData)
+
+        # TODO: remove this parameter. Arista VMs can be read by ptf from peer_dev_info.
+        self.rebootData['arista_vms'] = [
+            attr['mgmt_addr'] for dev, attr in testData['peer_dev_info'].items() if attr['hwsku'] == 'Arista-VM'
+        ]
+        self.hostMaxLen = len(self.rebootData['arista_vms']) - 1
+
+        testDataFiles = [{'source': source, 'name': name} for name, source in testData.items()]
+        self.__transferTestDataFiles(testDataFiles, self.ptfhost)
+
+    def __verifyRebootOper(self, rebootOper):
+        if isinstance(rebootOper, SadOperation):
+            logger.info('Running verify handler for reboot operation {}'.format(rebootOper))
+            rebootOper.verify()
+
+    def __revertRebootOper(self, rebootOper):
+        if isinstance(rebootOper, SadOperation):
+            logger.info('Running revert handler for reboot operation {}'.format(rebootOper))
+            rebootOper.revert()
+
     def __runPtfRunner(self, rebootOper=None):
         '''
         Run single PTF advanced-reboot.ReloadTest
         @param rebootOper:Reboot operation to conduct before/during reboot process
         '''
         logger.info("Running PTF runner on PTF host: {0}".format(self.ptfhost))
 
-        # Non-routing neighbor/dut lag/bgp, vlan port up/down operation is performed before dut reboot process
-        # lack of routing indicates it is preboot operation
-        prebootOper = rebootOper if rebootOper is not None and 'routing' not in rebootOper else None
-        # Routing add/remove is performed during dut reboot process
-        # presence of routing in reboot operation indicates it is during reboot operation (inboot)
-        inbootOper = rebootOper if rebootOper is not None and 'routing' in rebootOper else None
+        params={
+            "dut_username" : self.rebootData['dut_username'],
+            "dut_password" : self.rebootData['dut_password'],
+            "dut_hostname" : self.rebootData['dut_hostname'],
+            "reboot_limit_in_seconds" : self.rebootLimit,
+            "reboot_type" : self.rebootType,
+            "portchannel_ports_file" : self.rebootData['portchannel_interfaces_file'],
+            "vlan_ports_file" : self.rebootData['vlan_interfaces_file'],
+            "ports_file" : self.rebootData['ports_file'],
+            "dut_mac" : self.rebootData['dut_mac'],
+            "default_ip_range" : self.rebootData['default_ip_range'],
+            "vlan_ip_range" : self.rebootData['vlan_ip_range'],
+            "lo_v6_prefix" : self.rebootData['lo_v6_prefix'],
+            "arista_vms" : self.rebootData['arista_vms'],
+            "nexthop_ips" : self.rebootData['nexthop_ips'],
+            "allow_vlan_flooding" : self.allowVlanFlooding,
+            "sniff_time_incr" : self.sniffTimeIncr,
+            "setup_fdb_before_test" : True,
+            "vnet" : self.vnet,
+            "vnet_pkts" : self.vnetPkts,
+            "bgp_v4_v6_time_diff": self.bgpV4V6TimeDiff
+        }
+
+        if not isinstance(rebootOper, SadOperation):
+            # Non-routing neighbor/dut lag/bgp, vlan port up/down operation is performed before dut reboot process
+            # lack of routing indicates it is preboot operation
+            prebootOper = rebootOper if rebootOper is not None and 'routing' not in rebootOper else None
+            # Routing add/remove is performed during dut reboot process
+            # presence of routing in reboot operation indicates it is during reboot operation (inboot)
+            inbootOper = rebootOper if rebootOper is not None and 'routing' in rebootOper else None
+            params.update({
+                "preboot_files" : self.prebootFiles,
+                "preboot_oper" : prebootOper,
+                "inboot_oper" : inbootOper,
+            })
+        else:
+            params.update({'logfile_suffix': str(rebootOper)})
 
         self.__updateAndRestartArpResponder(rebootOper)
 
@@ -487,31 +540,7 @@ def __runPtfRunner(self, rebootOper=None):
             qlen=PTFRUNNER_QLEN,
             platform_dir="ptftests",
             platform="remote",
-            params={
-                "dut_username" : self.rebootData['dut_username'],
-                "dut_password" : self.rebootData['dut_password'],
-                "dut_hostname" : self.rebootData['dut_hostname'],
-                "reboot_limit_in_seconds" : self.rebootLimit,
-                "reboot_type" : self.rebootType,
-                "portchannel_ports_file" : self.rebootData['portchannel_interfaces_file'],
-                "vlan_ports_file" : self.rebootData['vlan_interfaces_file'],
-                "ports_file" : self.rebootData['ports_file'],
-                "dut_mac" : self.rebootData['dut_mac'],
-                "default_ip_range" : self.rebootData['default_ip_range'],
-                "vlan_ip_range" : self.rebootData['vlan_ip_range'],
-                "lo_v6_prefix" : self.rebootData['lo_v6_prefix'],
-                "arista_vms" : self.rebootData['arista_vms'],
-                "preboot_files" : self.prebootFiles,
-                "preboot_oper" : prebootOper,
-                "inboot_oper" : inbootOper,
-                "nexthop_ips" : self.rebootData['nexthop_ips'],
-                "allow_vlan_flooding" : self.allowVlanFlooding,
-                "sniff_time_incr" : self.sniffTimeIncr,
-                "setup_fdb_before_test" : True,
-                "vnet" : self.vnet,
-                "vnet_pkts" : self.vnetPkts,
-                "bgp_v4_v6_time_diff": self.bgpV4V6TimeDiff
-            },
+            params=params,
             log_file=u'/tmp/advanced-reboot.ReloadTest.log',
             module_ignore_errors=self.moduleIgnoreErrors
         )