[dualtor][active-active] Enable link down testcases (#6120)

Approach
What is the motivation for this PR?
Enable link down scenario for dualtor active-active testcases.

Signed-off-by: Longxiang Lyu [email protected]

How did you do it?
Modify port shutdown fixtures to shutdown ports based on the parameterized cable type.
Enable test_active_link_down_upstream

How did you verify/test it?
Failed because now linkmgrd fails toggle peer forwarding state, as sonic-net/sonic-linkmgrd#101

Any platform specific information?

tests/common/dualtor/control_plane_utils.py

@@ -140,7 +140,7 @@ def get_mismatched_ports(self, db):
 def verify_tor_states(
     expected_active_host, expected_standby_host,
     expected_standby_health='healthy', intf_names='all',
-    cable_type=CableType.default_type
+    cable_type=CableType.default_type, skip_state_db=False
 ):
     """
     Verifies that the expected states for active and standby ToRs are
@@ -150,20 +150,24 @@ def verify_tor_states(
         for duthost in expected_active_host:
             db_checker = DBChecker(duthost, 'active', 'healthy', intf_names=intf_names, cable_type=cable_type)
             db_checker.verify_db(APP_DB)
-            db_checker.verify_db(STATE_DB)
+            if not skip_state_db:
+                db_checker.verify_db(STATE_DB)
     elif expected_active_host is not None:
         duthost = expected_active_host
         db_checker = DBChecker(duthost, 'active', 'healthy', intf_names=intf_names, cable_type=cable_type)
         db_checker.verify_db(APP_DB)
-        db_checker.verify_db(STATE_DB)
+        if not skip_state_db:
+            db_checker.verify_db(STATE_DB)
 
     if isinstance(expected_standby_host, collections.Iterable):
         for duthost in expected_standby_host:
             db_checker = DBChecker(duthost, 'standby', expected_standby_health, intf_names=intf_names, cable_type=cable_type)
             db_checker.verify_db(APP_DB)
-            db_checker.verify_db(STATE_DB)
+            if not skip_state_db:
+                db_checker.verify_db(STATE_DB)
     elif expected_standby_host is not None:
         duthost = expected_standby_host
-        db_checker = DBChecker(duthost, 'standby', expected_standby_health, intf_names=intf_names)
+        db_checker = DBChecker(duthost, 'standby', expected_standby_health, intf_names=intf_names, cable_type=cable_type)
         db_checker.verify_db(APP_DB)
-        db_checker.verify_db(STATE_DB)
+        if not skip_state_db:
+            db_checker.verify_db(STATE_DB)

tests/common/dualtor/dual_tor_utils.py

@@ -21,6 +21,11 @@
 from tests.common.helpers.dut_ports import encode_dut_port_name
 from tests.common.dualtor.constants import UPPER_TOR, LOWER_TOR
 from tests.common.utilities import dump_scapy_packet_show_output, get_intf_by_sub_intf, is_ipv4_address
+from tests.common.dualtor.dual_tor_common import CableType
+from tests.common.dualtor.dual_tor_common import cable_type                                                     # lgtm[py/unused-import]
+from tests.common.dualtor.dual_tor_common import active_standby_ports                                           # lgtm[py/unused-import]
+from tests.common.dualtor.dual_tor_common import active_active_ports                                            # lgtm[py/unused-import]
+
 import ipaddress
 
 from ptf import mask
@@ -426,7 +431,7 @@ def _shutdown_fanout_tor_intfs(tor_host, tor_fanouthosts, tbinfo, dut_intfs=None
 
 
 @pytest.fixture
-def shutdown_fanout_upper_tor_intfs(upper_tor_host, upper_tor_fanouthosts, tbinfo):
+def shutdown_fanout_upper_tor_intfs(upper_tor_host, upper_tor_fanouthosts, tbinfo, cable_type, active_active_ports, active_standby_ports):
     """
     Fixture for shutting down fanout interfaces connected to specified upper_tor interfaces.
 
@@ -441,8 +446,12 @@ def shutdown_fanout_upper_tor_intfs(upper_tor_host, upper_tor_fanouthosts, tbinf
     shut_fanouts = []
     fanout_intfs_to_recover.clear()
 
+    mux_ports = active_active_ports if cable_type == CableType.active_active else active_standby_ports
+
     def shutdown(dut_intfs=None):
         logger.info('Shutdown fanout ports connected to upper_tor')
+        if dut_intfs is None:
+            dut_intfs = mux_ports
         shut_fanouts.append(_shutdown_fanout_tor_intfs(upper_tor_host, upper_tor_fanouthosts, tbinfo, dut_intfs))
 
     yield shutdown
@@ -455,7 +464,7 @@ def shutdown(dut_intfs=None):
 
 
 @pytest.fixture
-def shutdown_fanout_lower_tor_intfs(lower_tor_host, lower_tor_fanouthosts, tbinfo):
+def shutdown_fanout_lower_tor_intfs(lower_tor_host, lower_tor_fanouthosts, tbinfo, cable_type, active_active_ports, active_standby_ports):
     """
     Fixture for shutting down fanout interfaces connected to specified lower_tor interfaces.
 
@@ -470,8 +479,12 @@ def shutdown_fanout_lower_tor_intfs(lower_tor_host, lower_tor_fanouthosts, tbinf
     shut_fanouts = []
     fanout_intfs_to_recover.clear()
 
+    mux_ports = active_active_ports if cable_type == CableType.active_active else active_standby_ports
+
     def shutdown(dut_intfs=None):
         logger.info('Shutdown fanout ports connected to lower_tor')
+        if dut_intfs is None:
+            dut_intfs = mux_ports
         shut_fanouts.append(_shutdown_fanout_tor_intfs(lower_tor_host, lower_tor_fanouthosts, tbinfo, dut_intfs))
 
     yield shutdown
@@ -484,7 +497,7 @@ def shutdown(dut_intfs=None):
 
 
 @pytest.fixture
-def shutdown_fanout_tor_intfs(upper_tor_host, upper_tor_fanouthosts, lower_tor_host, lower_tor_fanouthosts, tbinfo):
+def shutdown_fanout_tor_intfs(upper_tor_host, upper_tor_fanouthosts, lower_tor_host, lower_tor_fanouthosts, tbinfo, cable_type, active_active_ports, active_standby_ports):
     """Fixture for shutting down fanout interfaces connected to specified lower_tor interfaces.
 
     Args:
@@ -500,11 +513,16 @@ def shutdown_fanout_tor_intfs(upper_tor_host, upper_tor_fanouthosts, lower_tor_h
     down_intfs = []
     fanout_intfs_to_recover.clear()
 
+    mux_ports = active_active_ports if cable_type == CableType.active_active else active_standby_ports
+
     def shutdown(dut_intfs=None, upper=False, lower=False):
         if not upper and not lower:
             logger.info('lower=False and upper=False, no fanout interface will be shutdown.')
             return
 
+        if dut_intfs is None:
+            dut_intfs = mux_ports
+
         if upper:
             logger.info('Shutdown fanout ports connected to upper_tor')
             down_intfs.extend(_shutdown_fanout_tor_intfs(upper_tor_host, upper_tor_fanouthosts, tbinfo, dut_intfs))

tests/dualtor_io/test_link_failure.py

@@ -8,31 +8,50 @@
 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_upper_tor                                                  # lgtm[py/unused-import]
 from tests.common.fixtures.ptfhost_utils import run_icmp_responder, run_garp_service, copy_ptftests_directory, change_mac_addresses             # lgtm[py/unused-import]
 from tests.common.dualtor.constants import MUX_SIM_ALLOWED_DISRUPTION_SEC
-from tests.common.dualtor.dual_tor_common import cable_type 
+from tests.common.dualtor.dual_tor_common import cable_type
+from tests.common.dualtor.dual_tor_common import CableType
+
 
 pytestmark = [
     pytest.mark.topology("dualtor")
 ]
 
 
+@pytest.mark.enable_active_active
 def test_active_link_down_upstream(
     upper_tor_host, lower_tor_host, send_server_to_t1_with_action,
     toggle_all_simulator_ports_to_upper_tor,
-    shutdown_fanout_upper_tor_intfs
+    shutdown_fanout_upper_tor_intfs, cable_type
 ):
     """
     Send traffic from server to T1 and shutdown the active ToR link.
     Verify switchover and disruption lasts < 1 second
     """
-    send_server_to_t1_with_action(
-        upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
-        allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs
-    )
-    verify_tor_states(
-        expected_active_host=lower_tor_host,
-        expected_standby_host=upper_tor_host,
-        expected_standby_health='unhealthy'
-    )
+    if cable_type == CableType.active_active:
+        send_server_to_t1_with_action(
+            upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
+            allowed_disruption=1, action=shutdown_fanout_upper_tor_intfs
+        )
+        verify_tor_states(
+            expected_active_host=lower_tor_host,
+            expected_standby_host=upper_tor_host,
+            expected_standby_health='unhealthy',
+            cable_type=cable_type,
+            skip_state_db=True
+        )
+
+    if cable_type == CableType.active_standby:
+        send_server_to_t1_with_action(
+            upper_tor_host, verify=True, delay=MUX_SIM_ALLOWED_DISRUPTION_SEC,
+            allowed_disruption=3, action=shutdown_fanout_upper_tor_intfs
+        )
+
+        verify_tor_states(
+            expected_active_host=lower_tor_host,
+            expected_standby_host=upper_tor_host,
+            expected_standby_health='unhealthy',
+            cable_type=cable_type,
+        )
 
 
 def test_active_link_down_downstream_active(