[sanity check]: Add mux simulator sanity check

tests/common/dualtor/__init__.py

@@ -0,0 +1,2 @@
+from tests.common.dualtor.dual_tor_utils import *
+from tests.common.dualtor.mux_simulator_control import *

tests/common/dualtor/dual_tor_utils.py

@@ -10,6 +10,8 @@
 from tests.common.helpers.assertions import pytest_assert as pt_assert
 from tests.common.helpers.dut_ports import encode_dut_port_name
 
+__all__ = ['tor_mux_intf', 'ptf_server_intf', 't1_upper_tor_intfs', 't1_lower_tor_intfs', 'upper_tor_host', 'lower_tor_host']
+
 logger = logging.getLogger(__name__)
 
 UPPER_TOR = 'upper_tor'
@@ -76,7 +78,7 @@ def lower_tor_host(duthosts):
 
     Uses the convention that the second ToR listed in the testbed file is the lower ToR
     '''
-    dut = duthosts[1]
+    dut = duthosts[-1]
     logger.info("Using {} as lower ToR".format(dut.hostname))
     return dut
 

tests/common/dualtor/mux_simulator_control.py

@@ -5,6 +5,10 @@
 from tests.common.helpers.assertions import pytest_assert
 from tests.common import utilities
 
+__all__ = ['check_simulator_read_side', 'mux_server_url', 'url', 'recover_all_directions', 'set_drop', 'set_output', 'toggle_all_simulator_ports_to_another_side', \
+           'toggle_all_simulator_ports_to_lower_tor', 'toggle_all_simulator_ports_to_random_side', 'toggle_all_simulator_ports_to_upper_tor', \
+           'toggle_simulator_port_to_lower_tor', 'toggle_simulator_port_to_upper_tor']
+
 logger = logging.getLogger(__name__)
 
 UPPER_TOR = "upper_tor"
@@ -19,7 +23,7 @@
 DROP = "drop"
 OUTPUT = "output"
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope='session')
 def mux_server_url(request, tbinfo):
     """
     A session level fixture to retrieve the address of mux simulator address
@@ -36,7 +40,7 @@ def mux_server_url(request, tbinfo):
     port = utilities.get_group_visible_vars(inv_files, server, 'mux_simulator_port')
     return "http://{}:{}/mux/{}".format(ip, port, vmset_name)
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def url(mux_server_url, duthost):
     """
     A helper function is returned to make fixture accept arguments
@@ -115,7 +119,7 @@ def _post(server_url, data):
         return False
     return True
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def set_drop(url):
     """
     A helper function is returned to make fixture accept arguments
@@ -135,7 +139,7 @@ def _set_drop(interface_name, directions):
 
     return _set_drop
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def set_output(url):
     """
     A helper function is returned to make fixture accept arguments
@@ -155,7 +159,7 @@ def _set_output(interface_name, directions):
 
     return _set_output
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def toggle_simulator_port_to_upper_tor(url):
     """
     Returns _toggle_simulator_port_to_upper_tor to make fixture accept arguments
@@ -173,7 +177,7 @@ def _toggle_simulator_port_to_upper_tor(interface_name):
 
     return _toggle_simulator_port_to_upper_tor
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def toggle_simulator_port_to_lower_tor(url):
     """
     Returns _toggle_simulator_port_to_lower_tor to make fixture accept arguments
@@ -190,7 +194,7 @@ def _toggle_simulator_port_to_lower_tor(interface_name):
 
     return _toggle_simulator_port_to_lower_tor
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def recover_all_directions(url):
     """
     A function level fixture, will return _recover_all_directions to make fixture accept arguments
@@ -209,7 +213,7 @@ def _recover_all_directions(interface_name):
 
     return _recover_all_directions
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def check_simulator_read_side(url):
     """
     A function level fixture, will return _check_simulator_read_side
@@ -238,7 +242,7 @@ def _check_simulator_read_side(interface_name):
 
     return _check_simulator_read_side
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def get_active_torhost(upper_tor_host, lower_tor_host, check_simulator_read_side):
     """
     A function level fixture which returns a helper function
@@ -313,7 +317,7 @@ def toggle_all_simulator_ports_to_random_side(mux_server_url):
     """
     _toggle_all_simulator_ports(mux_server_url, RANDOM)
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def simulator_server_down(set_drop, set_output):
     """
     A fixture to set drop on a given mux cable

tests/common/plugins/sanity_check/__init__.py

@@ -18,8 +18,31 @@
 
 logger = logging.getLogger(__name__)
 
-SUPPORTED_CHECKS = [member[0].replace('check_', '') for member in getmembers(checks, isfunction)
-                    if member[0].startswith('check_')]
+def is_check_item(member):
+    '''
+    Function to filter for valid check items
+
+    Used in conjuction with inspect.getmembers to make sure that only valid check functions/fixtures executed 
+
+    Valid check items must meet the following criteria:
+    - Is a function
+    - Is defined directly in sanity_checks/checks.py, NOT imported from another file
+    - Begins with the string 'check_'
+
+    Args:
+        member (object): The object to checked
+    Returns:
+        (bool) True if 'member' is a valid check function, False otherwise
+    '''
+    if isfunction(member):
+        in_check_file = member.__module__ == 'tests.common.plugins.sanity_check.checks'
+        starts_with_check = member.__name__.startswith('check_')
+        return in_check_file and starts_with_check
+    else:
+        return False
+
+
+SUPPORTED_CHECKS = [member[0].replace('check_', '') for member in getmembers(checks, is_check_item)]
 
 
 def _item2fixture(item):
@@ -127,6 +150,9 @@ def sanity_check(localhost, duthosts, request, fanouthosts, tbinfo):
     if tbinfo['topo']['type'] == 'ptf' and 'bgp' in check_items:
         check_items.remove('bgp')
 
+    if 'dualtor' not in tbinfo['topo']['name']:
+        check_items.remove('mux_simulator')
+
     logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \
         (skip_sanity, check_items, allow_recover, recover_method, post_check))
 

tests/common/plugins/sanity_check/checks.py

@@ -1,11 +1,15 @@
 import re
 import json
 import logging
-import time
-
+import ptf.testutils as testutils
 import pytest
+import time
 
+from ipaddress import ip_network, IPv4Network
+from tests.common.helpers.assertions import pytest_assert
 from tests.common.utilities import wait, wait_until
+from tests.common.dualtor.mux_simulator_control import *
+from tests.common.dualtor.dual_tor_utils import *
 
 logger = logging.getLogger(__name__)
 SYSTEM_STABILIZE_MAX_TIME = 300
@@ -18,7 +22,8 @@
     'check_bgp',
     'check_dbmemory',
     'check_monit',
-    'check_processes']
+    'check_processes',
+    'check_mux_simulator']
 
 
 @pytest.fixture(scope="module")
@@ -278,6 +283,170 @@ def _check_monit_services_status(check_result, monit_services_status):
     return check_result
 
 
+def get_arp_pkt_info(dut):
+    intf_mac = dut.facts['router_mac']
+    mgmt_ipv4 = None
+
+    mgmt_intf_facts = dut.get_running_config_facts()['MGMT_INTERFACE']
+
+    for mgmt_intf in mgmt_intf_facts:
+        for mgmt_ip in mgmt_intf_facts[mgmt_intf]:
+            if type(ip_network(mgmt_ip, strict=False)) is IPv4Network:
+                mgmt_ipv4 = mgmt_ip.split('/')[0]
+                return intf_mac, mgmt_ipv4
+
+    return intf_mac, mgmt_ipv4
+
+
+@pytest.fixture(scope='module')
+def check_mux_simulator(ptf_server_intf, tor_mux_intf, ptfadapter, upper_tor_host, lower_tor_host, \
+                        recover_all_directions, toggle_simulator_port_to_upper_tor, toggle_simulator_port_to_lower_tor, check_simulator_read_side):
+
+    def _check():
+        """
+        @summary: Checks if the OVS bridge mux simulator is functioning correctly
+        @return: A dictionary containing the testing result of the PTF interface tested:
+            {
+                'failed': <True/False>,
+                'failed_reason': <reason string>,
+                'intf': '<PTF interface name> mux simulator'
+            }
+        """
+        results = {
+                    'failed': False,
+                    'failed_reason': '',
+                    'check_item': '{} mux simulator'.format(ptf_server_intf)
+                }
+
+        logger.info("Checking mux simulator status for PTF interface {}".format(ptf_server_intf))
+        ptf_port_index = int(ptf_server_intf.replace('eth', ''))
+        recover_all_directions(tor_mux_intf)
+
+        upper_tor_intf_mac, upper_tor_mgmt_ip = get_arp_pkt_info(upper_tor_host)
+        lower_tor_intf_mac, lower_tor_mgmt_ip = get_arp_pkt_info(lower_tor_host)
+
+        upper_tor_ping_tgt_ip = '10.10.10.1'
+        lower_tor_ping_tgt_ip = '10.10.10.2'
+        ptf_arp_tgt_ip = '10.10.10.3'
+        ping_cmd = 'ping -I {} {} -c 1 -W 1; true'
+
+        upper_tor_exp_pkt = testutils.simple_arp_packet(eth_dst='ff:ff:ff:ff:ff:ff',
+                                                        eth_src=upper_tor_intf_mac,
+                                                        ip_snd=upper_tor_mgmt_ip,
+                                                        ip_tgt=upper_tor_ping_tgt_ip,
+                                                        hw_snd=upper_tor_intf_mac)
+        lower_tor_exp_pkt = testutils.simple_arp_packet(eth_dst='ff:ff:ff:ff:ff:ff',
+                                                        eth_src=lower_tor_intf_mac,
+                                                        ip_snd=lower_tor_mgmt_ip,
+                                                        ip_tgt=lower_tor_ping_tgt_ip,
+                                                        hw_snd=lower_tor_intf_mac)
+
+        ptf_arp_pkt = testutils.simple_arp_packet(ip_tgt=ptf_arp_tgt_ip,
+                                                ip_snd=ptf_arp_tgt_ip,
+                                                arp_op=2)
+
+        # Clear ARP tables to start in consistent state
+        upper_tor_host.shell("ip neigh flush all")
+        lower_tor_host.shell("ip neigh flush all")
+
+        # Run tests with upper ToR active
+        toggle_simulator_port_to_upper_tor(tor_mux_intf)
+
+        try:
+            pytest_assert(check_simulator_read_side(tor_mux_intf) == 1)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Unable to switch active link to upper ToR'
+            return results
+
+        # Ping from both ToRs, expect only message from upper ToR to reach PTF
+        upper_tor_host.shell(ping_cmd.format(tor_mux_intf, upper_tor_ping_tgt_ip))
+        try:
+            testutils.verify_packet(ptfadapter, upper_tor_exp_pkt, ptf_port_index)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from active upper ToR not received'
+            return results
+
+        lower_tor_host.shell(ping_cmd.format(tor_mux_intf, lower_tor_ping_tgt_ip))
+        try:
+            testutils.verify_no_packet(ptfadapter, lower_tor_exp_pkt, ptf_port_index)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from standby lower ToR received'
+            return results
+
+        # Send dummy ARP packets from PTF to ToR. Ensure that ARP is learned on both ToRs
+        upper_tor_host.shell("ip neigh flush all")
+        lower_tor_host.shell("ip neigh flush all")
+        testutils.send_packet(ptfadapter, ptf_port_index, ptf_arp_pkt)
+
+        upper_tor_arp_table = upper_tor_host.switch_arptable()['ansible_facts']['arptable']['v4']
+        lower_tor_arp_table = lower_tor_host.switch_arptable()['ansible_facts']['arptable']['v4']
+        try:
+            pytest_assert(ptf_arp_tgt_ip in upper_tor_arp_table)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from PTF not received on active upper ToR'
+            return results
+
+        try:
+            pytest_assert(ptf_arp_tgt_ip in lower_tor_arp_table)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from PTF not received on standby lower ToR'
+            return results
+
+        # Repeat all tests with lower ToR active
+        toggle_simulator_port_to_lower_tor(tor_mux_intf)
+        try:
+            pytest_assert(check_simulator_read_side(tor_mux_intf) == 2)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Unable to switch active link to lower ToR'
+            return results
+
+        lower_tor_host.shell(ping_cmd.format(tor_mux_intf, lower_tor_ping_tgt_ip))
+        try:
+            testutils.verify_packet(ptfadapter, lower_tor_exp_pkt, ptf_port_index)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from active lower ToR not received'
+            return results
+
+        upper_tor_host.shell(ping_cmd.format(tor_mux_intf, upper_tor_ping_tgt_ip))
+        try:
+            testutils.verify_no_packet(ptfadapter, upper_tor_exp_pkt, ptf_port_index)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from standby upper ToR received'
+            return results
+
+        upper_tor_host.shell("ip neigh flush all")
+        lower_tor_host.shell("ip neigh flush all")
+        testutils.send_packet(ptfadapter, ptf_port_index, ptf_arp_pkt)
+
+        upper_tor_arp_table = upper_tor_host.switch_arptable()['ansible_facts']['arptable']['v4']
+        lower_tor_arp_table = lower_tor_host.switch_arptable()['ansible_facts']['arptable']['v4']
+        try:
+            pytest_assert(ptf_arp_tgt_ip in upper_tor_arp_table)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from PTF not received on standby upper ToR'
+            return results
+
+        try:
+            pytest_assert(ptf_arp_tgt_ip in lower_tor_arp_table)
+        except AssertionError:
+            results['failed'] = True
+            results['failed_reason'] = 'Packet from PTF not received on active lower ToR'
+            return results
+
+        logger.info('Finished mux simulator check')
+        return results
+    return _check
+
+
 @pytest.fixture(scope="module")
 def check_monit(duthosts):
     """

tests/conftest.py

@@ -41,6 +41,7 @@
                   'tests.common.plugins.test_completeness',
                   'tests.common.plugins.log_section_start',
                   'tests.common.plugins.custom_fixtures',
+                  'tests.common.dualtor',
                   'tests.vxlan')