diff --git a/ansible/roles/test/files/ptftests/advanced-reboot.py b/ansible/roles/test/files/ptftests/advanced-reboot.py index e278089dc12..3f40108c776 100644 --- a/ansible/roles/test/files/ptftests/advanced-reboot.py +++ b/ansible/roles/test/files/ptftests/advanced-reboot.py @@ -156,6 +156,7 @@ def __init__(self): self.check_param('vnet_pkts', None, required=False) self.check_param('target_version', '', required=False) self.check_param('bgp_v4_v6_time_diff', 40, required=False) + self.check_param('logfile_suffix', None, required=False) if not self.test_params['preboot_oper'] or self.test_params['preboot_oper'] == 'None': self.test_params['preboot_oper'] = None if not self.test_params['inboot_oper'] or self.test_params['inboot_oper'] == 'None': @@ -167,9 +168,14 @@ def __init__(self): else: self.sad_oper = self.test_params['inboot_oper'] - if self.sad_oper: - self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.sad_oper) - self.report_file_name = '/tmp/%s-%s.json' % (self.test_params['reboot_type'], self.sad_oper) + if self.test_params['logfile_suffix']: + self.logfile_suffix = self.test_params['logfile_suffix'] + else: + self.logfile_suffix = self.sad_oper + + if self.logfile_suffix: + self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.logfile_suffix) + self.report_file_name = '/tmp/%s-%s.json' % (self.test_params['reboot_type'], self.logfile_suffix) else: self.log_file_name = '/tmp/%s.log' % self.test_params['reboot_type'] self.report_file_name = '/tmp/%s-report.json' % self.test_params['reboot_type'] @@ -354,7 +360,7 @@ def generate_arp_responder_conf(self, vlan_host_map): def dump_arp_responder_config(self, dump): # save data for arp_replay process - filename = "/tmp/from_t1.json" if self.sad_oper is None else "/tmp/from_t1_%s.json" % self.sad_oper + filename = "/tmp/from_t1.json" if self.logfile_suffix is None else "/tmp/from_t1_%s.json" % self.logfile_suffix with open(filename, "w") as fp: json.dump(dump, fp) @@ -1356,7 +1362,7 @@ def sniff_in_background(self, wait = None): self.sniffer_started.clear() def save_sniffed_packets(self): - filename = "/tmp/capture_%s.pcap" % self.sad_oper if self.sad_oper is not None else "/tmp/capture.pcap" + filename = "/tmp/capture_%s.pcap" % self.logfile_suffix if self.logfile_suffix is not None else "/tmp/capture.pcap" if self.packets: scapyall.wrpcap(filename, self.packets) self.log("Pcap file dumped to %s" % filename) @@ -1510,7 +1516,7 @@ def examine_flow(self, filename = None): self.log("Gaps in forwarding not found.") self.log("Total incoming packets captured %d" % received_counter) if packets: - filename = '/tmp/capture_filtered.pcap' if self.sad_oper is None else "/tmp/capture_filtered_%s.pcap" % self.sad_oper + filename = '/tmp/capture_filtered.pcap' if self.logfile_suffix is None else "/tmp/capture_filtered_%s.pcap" % self.logfile_suffix scapyall.wrpcap(filename, packets) self.log("Filtered pcap dumped to %s" % filename) diff --git a/tests/common/devices/sonic.py b/tests/common/devices/sonic.py index 57ba5c621a2..90a531eb502 100644 --- a/tests/common/devices/sonic.py +++ b/tests/common/devices/sonic.py @@ -769,6 +769,7 @@ def shutdown(self, ifname): Args: ifname: the interface to shutdown """ + logging.info("Shutting down {}".format(ifname)) return self.command("sudo config interface shutdown {}".format(ifname)) def shutdown_multiple(self, ifnames): @@ -788,6 +789,7 @@ def no_shutdown(self, ifname): Args: ifname: the interface to bring up """ + logging.info("Starting up {}".format(ifname)) return self.command("sudo config interface startup {}".format(ifname)) def no_shutdown_multiple(self, ifnames): diff --git a/tests/common/fixtures/advanced_reboot.py b/tests/common/fixtures/advanced_reboot.py index ff2da553e26..60f0e594f53 100644 --- a/tests/common/fixtures/advanced_reboot.py +++ b/tests/common/fixtures/advanced_reboot.py @@ -1,3 +1,4 @@ +import copy import ipaddress import itertools import json @@ -8,6 +9,7 @@ from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice from tests.common.platform.ssh_utils import prepare_testbed_ssh_keys as prepareTestbedSshKeys from tests.common.reboot import reboot as rebootDut +from tests.common.helpers.sad_path import SadOperation from tests.ptf_runner import ptf_runner logger = logging.getLogger(__name__) @@ -199,6 +201,10 @@ def __validateAndBuildSadList(self): sadList = [item for item in itertools.chain(prebootList, inbootList)] for item in sadList: + # TODO: Move all sad path logic out of ptf script to pytest. + # Once done, we can make a sad_operation fixture. + if isinstance(item, SadOperation): + continue if ':' not in item: continue itemCnt = int(item.split(':')[-1]) @@ -212,16 +218,6 @@ def __validateAndBuildSadList(self): 'Lag count is greater than or equal to number of VM hosts. ' 'Current val = {0} Max val = {1}' ).format(itemCnt, self.hostMaxLen) - if 'lag_member_down' in item: - assert itemCnt <= self.lagMemberCnt, ( - 'Lag member count is greater than available number of lag members. ' - 'Current val = {0} Available cnt = {1}' - ).format(itemCnt, self.lagMemberCnt) - if 'vlan_port_down' in item: - assert itemCnt <= self.vlanMaxCnt, ( - 'Vlan count is greater than or equal to number of Vlan interfaces. ' - 'Current val = {0} Max val = {1}' - ).format(itemCnt, self.vlanMaxCnt) if 'routing' in item: assert itemCnt <= self.hostMaxCnt, ( 'Number of prefixes is greater than allowed max. ' @@ -330,15 +326,6 @@ def __setupTestbed(self): ''' Sets testbed up. It tranfers test data files, ARP responder, and runs script to update IPs and MAC addresses. ''' - testDataFiles = [ - {'source' : self.mgFacts['minigraph_portchannels'], 'name' : 'portchannel_interfaces'}, - {'source' : self.mgFacts['minigraph_vlans'], 'name' : 'vlan_interfaces' }, - {'source' : self.mgFacts['minigraph_ptf_indices'], 'name' : 'ports' }, - {'source' : self.mgFacts['minigraph_devices'], 'name' : 'peer_dev_info' }, - {'source' : self.mgFacts['minigraph_neighbors'], 'name' : 'neigh_port_info' }, - ] - self.__transferTestDataFiles(testDataFiles, self.ptfhost) - self.__runScript(['remove_ip.sh'], self.ptfhost) self.__prepareTestbedSshKeys() @@ -441,11 +428,14 @@ def runRebootTest(self): for rebootOper in self.rebootData['sadList']: count += 1 try: + self.__setupRebootOper(rebootOper) result = self.__runPtfRunner(rebootOper) + self.__verifyRebootOper(rebootOper) finally: # always capture the test logs self.__fetchTestLogs(rebootOper) self.__clearArpAndFdbTables() + self.__revertRebootOper(rebootOper) if not result: return result if len(self.rebootData['sadList']) > 1 and count != len(self.rebootData['sadList']): @@ -463,6 +453,38 @@ def runRebootTestcase(self, prebootList=None, inbootList=None, prebootFiles=None self.imageInstall(prebootList, inbootList, prebootFiles) return self.runRebootTest() + def __setupRebootOper(self, rebootOper): + testData = { + 'portchannel_interfaces': copy.deepcopy(self.mgFacts['minigraph_portchannels']), + 'vlan_interfaces': copy.deepcopy(self.mgFacts['minigraph_vlans']), + 'ports': copy.deepcopy(self.mgFacts['minigraph_ptf_indices']), + 'peer_dev_info': copy.deepcopy(self.mgFacts['minigraph_devices']), + 'neigh_port_info': copy.deepcopy(self.mgFacts['minigraph_neighbors']), + } + + if isinstance(rebootOper, SadOperation): + logger.info('Running setup handler for reboot operation {}'.format(rebootOper)) + rebootOper.setup(testData) + + # TODO: remove this parameter. Arista VMs can be read by ptf from peer_dev_info. + self.rebootData['arista_vms'] = [ + attr['mgmt_addr'] for dev, attr in testData['peer_dev_info'].items() if attr['hwsku'] == 'Arista-VM' + ] + self.hostMaxLen = len(self.rebootData['arista_vms']) - 1 + + testDataFiles = [{'source': source, 'name': name} for name, source in testData.items()] + self.__transferTestDataFiles(testDataFiles, self.ptfhost) + + def __verifyRebootOper(self, rebootOper): + if isinstance(rebootOper, SadOperation): + logger.info('Running verify handler for reboot operation {}'.format(rebootOper)) + rebootOper.verify() + + def __revertRebootOper(self, rebootOper): + if isinstance(rebootOper, SadOperation): + logger.info('Running revert handler for reboot operation {}'.format(rebootOper)) + rebootOper.revert() + def __runPtfRunner(self, rebootOper=None): ''' Run single PTF advanced-reboot.ReloadTest @@ -470,12 +492,43 @@ def __runPtfRunner(self, rebootOper=None): ''' logger.info("Running PTF runner on PTF host: {0}".format(self.ptfhost)) - # Non-routing neighbor/dut lag/bgp, vlan port up/down operation is performed before dut reboot process - # lack of routing indicates it is preboot operation - prebootOper = rebootOper if rebootOper is not None and 'routing' not in rebootOper else None - # Routing add/remove is performed during dut reboot process - # presence of routing in reboot operation indicates it is during reboot operation (inboot) - inbootOper = rebootOper if rebootOper is not None and 'routing' in rebootOper else None + params={ + "dut_username" : self.rebootData['dut_username'], + "dut_password" : self.rebootData['dut_password'], + "dut_hostname" : self.rebootData['dut_hostname'], + "reboot_limit_in_seconds" : self.rebootLimit, + "reboot_type" : self.rebootType, + "portchannel_ports_file" : self.rebootData['portchannel_interfaces_file'], + "vlan_ports_file" : self.rebootData['vlan_interfaces_file'], + "ports_file" : self.rebootData['ports_file'], + "dut_mac" : self.rebootData['dut_mac'], + "default_ip_range" : self.rebootData['default_ip_range'], + "vlan_ip_range" : self.rebootData['vlan_ip_range'], + "lo_v6_prefix" : self.rebootData['lo_v6_prefix'], + "arista_vms" : self.rebootData['arista_vms'], + "nexthop_ips" : self.rebootData['nexthop_ips'], + "allow_vlan_flooding" : self.allowVlanFlooding, + "sniff_time_incr" : self.sniffTimeIncr, + "setup_fdb_before_test" : True, + "vnet" : self.vnet, + "vnet_pkts" : self.vnetPkts, + "bgp_v4_v6_time_diff": self.bgpV4V6TimeDiff + } + + if not isinstance(rebootOper, SadOperation): + # Non-routing neighbor/dut lag/bgp, vlan port up/down operation is performed before dut reboot process + # lack of routing indicates it is preboot operation + prebootOper = rebootOper if rebootOper is not None and 'routing' not in rebootOper else None + # Routing add/remove is performed during dut reboot process + # presence of routing in reboot operation indicates it is during reboot operation (inboot) + inbootOper = rebootOper if rebootOper is not None and 'routing' in rebootOper else None + params.update({ + "preboot_files" : self.prebootFiles, + "preboot_oper" : prebootOper, + "inboot_oper" : inbootOper, + }) + else: + params.update({'logfile_suffix': str(rebootOper)}) self.__updateAndRestartArpResponder(rebootOper) @@ -487,31 +540,7 @@ def __runPtfRunner(self, rebootOper=None): qlen=PTFRUNNER_QLEN, platform_dir="ptftests", platform="remote", - params={ - "dut_username" : self.rebootData['dut_username'], - "dut_password" : self.rebootData['dut_password'], - "dut_hostname" : self.rebootData['dut_hostname'], - "reboot_limit_in_seconds" : self.rebootLimit, - "reboot_type" : self.rebootType, - "portchannel_ports_file" : self.rebootData['portchannel_interfaces_file'], - "vlan_ports_file" : self.rebootData['vlan_interfaces_file'], - "ports_file" : self.rebootData['ports_file'], - "dut_mac" : self.rebootData['dut_mac'], - "default_ip_range" : self.rebootData['default_ip_range'], - "vlan_ip_range" : self.rebootData['vlan_ip_range'], - "lo_v6_prefix" : self.rebootData['lo_v6_prefix'], - "arista_vms" : self.rebootData['arista_vms'], - "preboot_files" : self.prebootFiles, - "preboot_oper" : prebootOper, - "inboot_oper" : inbootOper, - "nexthop_ips" : self.rebootData['nexthop_ips'], - "allow_vlan_flooding" : self.allowVlanFlooding, - "sniff_time_incr" : self.sniffTimeIncr, - "setup_fdb_before_test" : True, - "vnet" : self.vnet, - "vnet_pkts" : self.vnetPkts, - "bgp_v4_v6_time_diff": self.bgpV4V6TimeDiff - }, + params=params, log_file=u'/tmp/advanced-reboot.ReloadTest.log', module_ignore_errors=self.moduleIgnoreErrors ) diff --git a/tests/common/helpers/sad_path.py b/tests/common/helpers/sad_path.py new file mode 100644 index 00000000000..e61cf4570c3 --- /dev/null +++ b/tests/common/helpers/sad_path.py @@ -0,0 +1,280 @@ +""" Module contains sad path operations. """ + +import datetime +import logging + +from itertools import groupby, chain, islice + +from tests.common.platform.device_utils import fanout_switch_port_lookup + +logger = logging.getLogger(__name__) + + +class SadOperation(object): + """ SadOperation interface class. """ + + def setup(self, test_data): + """ Perform sad path setup operations and modify the test_data + passed to PTF script accordingly. """ + raise NotImplementedError + + # TODO: split verification into to phases - preboot verify and postboot verify. + # Currently there is no verification in sad_path.py done prior to warm-reboot. + # So it could be a race when sad operation takes a while to affect the testbed + # but we are doing warm-reboot prior to that. Currently the preparation in + # advanced-reboot.py is long enough so that won't happen. + def verify(self): + """ Verify handler that runs after warm-reboot completes. + Checks sad path operataions result after warm-reboot. """ + raise NotImplementedError + + def revert(self): + """ Revert changes done in setup. """ + raise NotImplementedError + + +class Selector(object): + """ Selector interface provides a select() method + to choose test objects from the input list. """ + + def select(self, objlist): + """ Choose test objects from objlist. """ + raise NotImplementedError + + +class PhyPropsPortSelector(Selector): + """ Select the port based on physical port settings. """ + + def __init__(self, duthost, count): + self.duthost = duthost + self.count = count + + def select(self, objlist): + port_table = self.duthost.get_running_config_facts()["PORT"] + + def group_func(port): + _, attrs = port + width = len(attrs["lanes"].split(",")) + speed = attrs.get("speed") + fec = attrs.get("fec") + + return width, speed, fec + + # For ports passed to this method group them by width, speed and fec + # and choose number of ports taking each from a different group. + # If is greater then the number of groups start over till we fill + # the output list with the number of ports requested. + # Assertion is raised when there are no enough ports. + port_items = [(name, attrs) for name, attrs in port_table.items() if name in objlist] + assert len(port_items) >= self.count, "No enough ports to test, required at least {}".format(self.count) + groups = [list(group) for _, group in groupby(sorted(port_items, key=group_func), key=group_func)] + return [name for name, _ in islice(chain.from_iterable(zip(*groups)), self.count)] + + +class DatetimeSelector(Selector): + """ Select from list based on current datetime. """ + + def __init__(self, count): + self.count = count + + def select(self, objlist): + assert len(objlist) >= self.count, "Not enough elements, required at least {}".format(self.count) + # Get some start index and select items from the list + # starting from index till the end, if the amount is less then + # self.count it will fill the rest starting from the beginning of + # the list. + index = datetime.datetime.now().day % len(objlist) + selected = (objlist[index:] + objlist[:index])[:self.count] + return selected + + +class VlanMemberDown(SadOperation): + """ Base class for vlan member down scenarios. """ + + def __init__(self, duthost, port_selector): + self.duthost = duthost + self.ports = port_selector.select(duthost.get_vlan_intfs()) + + def setup(self, test_data): + vlans = test_data["vlan_interfaces"] + # Exclude down vlan members + for vlan in vlans.values(): + vlan["members"] = list(set(vlan["members"]) - set(self.ports)) + + +class DutVlanMemberDown(VlanMemberDown): + """ Sad path test case to verify warm-reboot when vlan member port goes administartively down. """ + + def __init__(self, duthost, port_selector): + super(DutVlanMemberDown, self).__init__(duthost, port_selector) + logger.info("Selected ports for DUT vlan member down case {}".format(self.ports)) + + def setup(self, test_data): + super(DutVlanMemberDown, self).setup(test_data) + self.duthost.shutdown_multiple(self.ports) + + def verify(self): + facts = self.duthost.show_interface(command="status", interfaces=self.ports) + port_facts = facts["ansible_facts"]["int_status"] + assert all([port["admin_state"] == "down" for port in port_facts.values()]) + + def revert(self): + self.duthost.no_shutdown_multiple(self.ports) + + def __str__(self): + return "vlan_port_down:{}".format(len(self.ports)) + + +class NeighVlanMemberDown(VlanMemberDown): + """ Sad path test case to verify warm-reboot when vlan member port goes operationaly down + by shutting down the corresponding port on the neighbor side. """ + + def __init__(self, duthost, fanouthosts, port_selector): + super(NeighVlanMemberDown, self).__init__(duthost, port_selector) + self.fanouthosts = fanouthosts + logger.info("Selected ports for neighbor vlan member down case {}".format(self.ports)) + + def setup(self, test_data): + super(NeighVlanMemberDown, self).setup(test_data) + + for port in self.ports: + fanout, fanport = fanout_switch_port_lookup(self.fanouthosts, self.duthost.hostname, port) + fanout.shutdown(fanport) + + def verify(self): + facts = self.duthost.show_interface(command="status", interfaces=self.ports) + port_facts = facts["ansible_facts"]["int_status"] + assert all([port["oper_state"] == "down" for port in port_facts.values()]) + + def revert(self): + for port in self.ports: + fanout, fanport = fanout_switch_port_lookup(self.fanouthosts, self.duthost.hostname, port) + fanout.no_shutdown(fanport) + + def __str__(self): + return "neigh_vlan_member_down:{}".format(len(self.ports)) + + +class LagMemberDown(SadOperation): + """ Base class for LAG member down sad path scenarios. """ + + def __init__(self, duthost, nbrhosts, vm_selector, port_selector): + super(LagMemberDown, self).__init__() + vms = {vm: nbrhosts[vm] for vm in vm_selector.select(list(nbrhosts))} + mg_facts = duthost.minigraph_facts(host=duthost.hostname)["ansible_facts"] + dut_port_to_neighbor = mg_facts["minigraph_neighbors"] + lags = mg_facts["minigraph_portchannels"] + + # Build neighbor hostname to DUT LAG mapping and + # DUT port to DUT LAG mapping. + neigh_to_lag = {} + port_to_lag = {} + for dut_port, neigh_info in dut_port_to_neighbor.items(): + for lag in lags.values(): + if dut_port in lag["members"]: + neigh_to_lag[neigh_info["name"]] = lag + port_to_lag[dut_port] = lag + + ports = [] + for vm in vms: + ports.extend(port_selector.select(neigh_to_lag[vm]["members"])) + + self.vms = vms + self.ports = ports + self.duthost = duthost + self.nbrhosts = nbrhosts + self.neigh_to_lag = neigh_to_lag + self.port_to_lag = port_to_lag + + def setup(self, test_data): + lags = test_data["portchannel_interfaces"] + peer_dev_info = test_data["peer_dev_info"] + + # Exclude down LAG members + for lag in lags.values(): + lag["members"] = list(set(lag["members"]) - set(self.ports)) + + # Exclude VMs corresponding to down LAGs + for vm in self.vms: + peer_dev_info.pop(vm) + + def verify(self): + lag_facts = self.duthost.lag_facts(host=self.duthost.hostname)["ansible_facts"]["lag_facts"] + port_facts = self.duthost.show_interface(command="status")["ansible_facts"]["int_status"] + + for port in self.ports: + lag = self.port_to_lag[port] + port_stats = lag_facts["lags"][lag["name"]]["po_stats"]["ports"][port] + assert not port_stats["runner"]["aggregator"]["selected"] + + for vm in self.vms: + assert port_facts[self.neigh_to_lag[vm]["name"]]["oper_state"] == "down" + nbrhost = self.nbrhosts[vm]["host"] + # TODO: remove this hardcode, implement a mapping of DUT LAG to VM LAG. + nbr_lag_name = "Port-Channel1" + commands = ["show interface {} | json".format(nbr_lag_name)] + output = nbrhost.eos_command(commands=commands)["stdout"][0] + state = output["interfaces"][nbr_lag_name]["interfaceStatus"] + assert state in ["notconnect"] + + +class DutLagMemberDown(LagMemberDown): + """ Sad path to test warm-reboot when LAG member on DUT is shutdown + and verify that after warm-reboot LAG member state is still down on DUT and neighbor. """ + + def __init__(self, duthost, nbrhosts, vm_selector, port_selector): + super(DutLagMemberDown, self).__init__(duthost, nbrhosts, vm_selector, port_selector) + logger.info("Selected ports for DUT LAG member down case {}".format(self.ports)) + + def setup(self, test_data): + super(DutLagMemberDown, self).setup(test_data) + self.duthost.shutdown_multiple(self.ports) + + def revert(self): + self.duthost.no_shutdown_multiple(self.ports) + + def __str__(self): + return "dut_lag_member_down:{}:{}".format(len(self.vms), len(self.ports)) + + +class NeighLagMemberDown(LagMemberDown): + """ Sad path to test warm-reboot when LAG member on neighbor is shutdown + and verify that after warm-reboot LAG member state is still down on DUT and neighbor. """ + + def __init__(self, duthost, nbrhosts, fanouthosts, vm_selector, port_selector): + super(NeighLagMemberDown, self).__init__(duthost, nbrhosts, vm_selector, port_selector) + logger.info("Selected ports for neighbor LAG member down case {}".format(self.ports)) + + mg_facts = self.duthost.minigraph_facts(host=self.duthost.hostname)["ansible_facts"] + mg_neighs = mg_facts["minigraph_neighbors"].items() + + self.fanouthosts = fanouthosts + self.dut_port_to_nbr = {port: nbr_info["name"] for port, nbr_info in mg_neighs} + self.dut_port_to_nbr_port = {port: nbr_info["port"] for port, nbr_info in mg_neighs} + + def setup(self, test_data): + super(NeighLagMemberDown, self).setup(test_data) + self._change_ports_state(bring_up=False) + + def revert(self): + self._change_ports_state(bring_up=True) + + def _change_ports_state(self, bring_up): + for port in self.ports: + nbrname = self.dut_port_to_nbr[port] + nbrport = self.dut_port_to_nbr_port[port] + nbrhost = self.nbrhosts[nbrname]["host"] + if bring_up: + nbrhost.no_shutdown(nbrport) + else: + nbrhost.shutdown(nbrport) + + fanout, fanport = fanout_switch_port_lookup(self.fanouthosts, self.duthost.hostname, port) + if bring_up: + fanout.no_shutdown(fanport) + else: + fanout.shutdown(fanport) + + + def __str__(self): + return "neigh_lag_member_down:{}:{}".format(len(self.vms), len(self.ports)) diff --git a/tests/platform_tests/test_advanced_reboot.py b/tests/platform_tests/test_advanced_reboot.py index 4da98e65e21..6a9cf8f65c7 100644 --- a/tests/platform_tests/test_advanced_reboot.py +++ b/tests/platform_tests/test_advanced_reboot.py @@ -6,6 +6,15 @@ from tests.platform_tests.verify_dut_health import verify_dut_health # lgtm[py/unused-import] from tests.platform_tests.verify_dut_health import add_fail_step_to_reboot # lgtm[py/unused-import] +from tests.common.helpers.sad_path import ( + DutVlanMemberDown, + NeighVlanMemberDown, + DutLagMemberDown, + NeighLagMemberDown, + PhyPropsPortSelector, + DatetimeSelector, +) + pytestmark = [ pytest.mark.disable_loganalyzer, pytest.mark.topology('t0') @@ -68,13 +77,11 @@ def test_cancelled_warm_reboot(request, add_fail_step_to_reboot, verify_dut_heal ### Tetcases to verify reboot procedure with SAD cases ### def test_warm_reboot_sad(request, get_advanced_reboot, verify_dut_health, - backup_and_restore_config_db, advanceboot_neighbor_restore): + backup_and_restore_config_db, advanceboot_neighbor_restore, + duthost, fanouthosts, nbrhosts): ''' Warm reboot with sad path - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' @@ -84,9 +91,14 @@ def test_warm_reboot_sad(request, get_advanced_reboot, verify_dut_health, 'dut_bgp_down', # Shutdown single BGP session on DUT brefore rebooting it 'dut_lag_down', # Shutdown single LAG session on DUT brefore rebooting it 'neigh_lag_down', # Shutdown single LAG session on remote device (VM) before reboot DUT - 'dut_lag_member_down:1:1', # Shutdown 1 LAG member corresponding to 1 remote device (VM) on DUT - 'neigh_lag_member_down:1:1', # Shutdown 1 LAG member on 1 remote device (VM) - 'vlan_port_down', # Shutdown 1 vlan port (interface) on DUT + # Shutdown 1 LAG member corresponding to 1 remote device (VM) on DUT + DutLagMemberDown(duthost, nbrhosts, DatetimeSelector(1), PhyPropsPortSelector(duthost, 1)), + # Shutdown 1 LAG member on 1 remote device (VM) + NeighLagMemberDown(duthost, nbrhosts, fanouthosts, DatetimeSelector(1), PhyPropsPortSelector(duthost, 1)), + # Shutdown 1 vlan port (interface) on DUT + DutVlanMemberDown(duthost, PhyPropsPortSelector(duthost, 1)), + # Shutdown 1 vlan port (interface) on fanout + NeighVlanMemberDown(duthost, fanouthosts, PhyPropsPortSelector(duthost, 1)), ] advancedReboot.runRebootTestcase( @@ -96,13 +108,11 @@ def test_warm_reboot_sad(request, get_advanced_reboot, verify_dut_health, def test_warm_reboot_multi_sad(request, get_advanced_reboot, verify_dut_health, - backup_and_restore_config_db, advanceboot_neighbor_restore): + backup_and_restore_config_db, advanceboot_neighbor_restore, + duthost, fanouthosts, nbrhosts): ''' Warm reboot with multi sad path - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' @@ -113,17 +123,20 @@ def test_warm_reboot_multi_sad(request, get_advanced_reboot, verify_dut_health, 'dut_bgp_down:3', # Shutdown 3 BGP sessions on DUT brefore rebooting it 'dut_lag_down:2', # Shutdown 2 LAG sessions on DUT brefore rebooting it 'neigh_lag_down:3', # Shutdown 1 LAG session on 3 remote devices (VMs) before reboot DUT - 'dut_lag_member_down:3:1', # Shutdown 1 LAG member of 3 LAG sessions corresponding to 3 remote devices (VM) - # on DUT - 'neigh_lag_member_down:2:1', # Shutdown 1 LAG member of 2 LAG sessions on 2 remote devices (VM) (1 each) - 'vlan_port_down:4', + # Shutdown 1 LAG member of 3 LAG sessions corresponding to 3 remote devices (VM) + # on DUT + DutLagMemberDown(duthost, nbrhosts, DatetimeSelector(3), PhyPropsPortSelector(duthost, 1)), + # Shutdown 1 LAG member of 2 LAG sessions on 2 remote devices (VM) (1 each) + NeighLagMemberDown(duthost, nbrhosts, fanouthosts, DatetimeSelector(2), PhyPropsPortSelector(duthost, 1)), + DutVlanMemberDown(duthost, PhyPropsPortSelector(duthost, 4)), + NeighVlanMemberDown(duthost, fanouthosts, PhyPropsPortSelector(duthost, 4)), ] + ([ - 'dut_lag_member_down:2:{0}'.format(lagMemberCnt), - # Shutdown LAG member(s) of 2 LAG sessions corresponding to 2 remote - # devices (VM) on DUT - 'neigh_lag_member_down:3:{0}'.format(lagMemberCnt), - # Shutdown LAG member(s) of 3 LAG sessions on 3 remote devices (VM) - # (1 each) + # Shutdown LAG member(s) of 2 LAG sessions corresponding to 2 remote + # devices (VM) on DUT + DutLagMemberDown(duthost, nbrhosts, DatetimeSelector(2), PhyPropsPortSelector(duthost, lagMemberCnt)), + # Shutdown LAG member(s) of 3 LAG sessions on 3 remote devices (VM) + # (1 each) + NeighLagMemberDown(duthost, nbrhosts, fanouthosts, DatetimeSelector(3), PhyPropsPortSelector(duthost, lagMemberCnt)), ] if advancedReboot.getTestbedType() in ['t0-64', 't0-116', 't0-64-32'] else []) advancedReboot.runRebootTestcase( @@ -159,9 +172,6 @@ def test_warm_reboot_sad_bgp(request, get_advanced_reboot, verify_dut_health, ''' Warm reboot with sad (bgp) - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' @@ -178,29 +188,29 @@ def test_warm_reboot_sad_bgp(request, get_advanced_reboot, verify_dut_health, def test_warm_reboot_sad_lag_member(request, get_advanced_reboot, verify_dut_health, - backup_and_restore_config_db, advanceboot_neighbor_restore): + backup_and_restore_config_db, advanceboot_neighbor_restore, + duthost, fanouthosts, nbrhosts): ''' Warm reboot with sad path (lag member) - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' advancedReboot = get_advanced_reboot(rebootType='warm-reboot') lagMemberCnt = advancedReboot.getlagMemberCnt() prebootList = [ - 'dut_lag_member_down:3:1', # Shutdown 1 LAG member of 3 LAG sessions corresponding to 3 remote devices (VM) - # on DUT - 'neigh_lag_member_down:2:1', # Shutdown 1 LAG member of 2 LAG sessions on 2 remote devices (VM) (1 each) + # Shutdown 1 LAG member of 3 LAG sessions corresponding to 3 remote devices (VM) + # on DUT + DutLagMemberDown(duthost, nbrhosts, DatetimeSelector(3), PhyPropsPortSelector(duthost, 1)), + # Shutdown 1 LAG member of 2 LAG sessions on 2 remote devices (VM) (1 each) + NeighLagMemberDown(duthost, nbrhosts, fanouthosts, DatetimeSelector(2), PhyPropsPortSelector(duthost, 1)), ] + ([ - 'dut_lag_member_down:2:{0}'.format(lagMemberCnt), - # Shutdown LAG member(s) of 2 LAG sessions corresponding to 2 remote - # devices (VM) on DUT - 'neigh_lag_member_down:3:{0}'.format(lagMemberCnt), - # Shutdown LAG member(s) of 3 LAG sessions on 3 remote devices (VM) - # (1 each) + # Shutdown LAG member(s) of 2 LAG sessions corresponding to 2 remote + # devices (VM) on DUT + DutLagMemberDown(duthost, nbrhosts, DatetimeSelector(2), PhyPropsPortSelector(duthost, lagMemberCnt)), + # Shutdown LAG member(s) of 3 LAG sessions on 3 remote devices (VM) + # (1 each) + NeighLagMemberDown(duthost, nbrhosts, fanouthosts, DatetimeSelector(3), PhyPropsPortSelector(duthost, lagMemberCnt)), ] if advancedReboot.getTestbedType() in ['t0-64', 't0-116', 't0-64-32'] else []) advancedReboot.runRebootTestcase( @@ -214,9 +224,6 @@ def test_warm_reboot_sad_lag(request, get_advanced_reboot, verify_dut_health, ''' Warm reboot with sad path (lag) - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' @@ -233,19 +240,17 @@ def test_warm_reboot_sad_lag(request, get_advanced_reboot, verify_dut_health, def test_warm_reboot_sad_vlan_port(request, get_advanced_reboot, verify_dut_health, - backup_and_restore_config_db): + backup_and_restore_config_db, duthost, fanouthosts): ''' Warm reboot with sad path (vlan port) - prebootList format is 'preboot oper type:number of VMS down:number of lag members down'. - For non lag member cases, this parameter will be skipped - @param request: Spytest commandline argument @param get_advanced_reboot: advanced reboot test fixture ''' advancedReboot = get_advanced_reboot(rebootType='warm-reboot') prebootList = [ - 'vlan_port_down:4', # Shutdown 4 vlan ports (interfaces) on DUT + DutVlanMemberDown(duthost, PhyPropsPortSelector(duthost, 4)), # Shutdown 4 vlan ports (interfaces) on DUT + NeighVlanMemberDown(duthost, fanouthosts, PhyPropsPortSelector(duthost, 4)), # Shutdown 4 vlan ports (interfaces) on fanout ] advancedReboot.runRebootTestcase(