Add DVS tests for everflow warm reboot support

tests/conftest.py

@@ -1000,16 +1000,25 @@ def remove_neighbor(self, interface, ip):
         tbl._del(interface + ":" + ip)
         time.sleep(1)
 
-    # deps: mirror_port_erspan
+    # deps: mirror_port_erspan, warm_reboot
     def add_route(self, prefix, nexthop):
         self.runcmd("ip route add " + prefix + " via " + nexthop)
         time.sleep(1)
 
-    # deps: mirror_port_erspan
+    # deps: mirror_port_erspan, warm_reboot
     def change_route(self, prefix, nexthop):
         self.runcmd("ip route change " + prefix + " via " + nexthop)
         time.sleep(1)
 
+    # deps: warm_reboot
+    def change_route_ecmp(self, prefix, nexthops):
+        cmd = ""
+        for nexthop in nexthops:
+            cmd += " nexthop via " + nexthop
+
+        self.runcmd("ip route change " + prefix + cmd)
+        time.sleep(1)
+
     # deps: acl, mirror_port_erspan
     def remove_route(self, prefix):
         self.runcmd("ip route del " + prefix)

tests/dvslib/dvs_acl.py

@@ -331,6 +331,11 @@ def remove_acl_rule(self, table_name: str, rule_name: str) -> None:
         """
         self.config_db.delete_entry("ACL_RULE", "{}|{}".format(table_name, rule_name))
 
+    def verify_acl_rule_count(self, expected: int) -> None:
+        """Verify that there are N rules in the ASIC DB."""
+        num_keys = len(self.asic_db.default_acl_entries)
+        self.asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", num_keys + expected)
+
     def verify_no_acl_rules(self) -> None:
         """Verify that there are no ACL rules in the ASIC DB."""
         num_keys = len(self.asic_db.default_acl_entries)

tests/dvslib/dvs_mirror.py

@@ -23,18 +23,21 @@ def create_span_session(self, name, dst_port, src_ports=None, direction="BOTH",
         self.config_db.create_entry("MIRROR_SESSION", name, mirror_entry)
 
     def create_erspan_session(self, name, src, dst, gre, dscp, ttl, queue, policer=None, src_ports=None, direction="BOTH"):
-        mirror_entry = {}
-        mirror_entry["src_ip"] = src
-        mirror_entry["dst_ip"] = dst
-        mirror_entry["gre_type"] = gre
-        mirror_entry["dscp"] = dscp
-        mirror_entry["ttl"] = ttl
-        mirror_entry["queue"] = queue
+        mirror_entry = {
+            "src_ip": src,
+            "dst_ip": dst,
+            "gre_type": gre,
+            "dscp": dscp,
+            "ttl": ttl,
+            "queue": queue,
+            "direction": direction
+        }
+
         if policer:
             mirror_entry["policer"] = policer
+
         if src_ports:
             mirror_entry["src_port"] = src_ports
-        mirror_entry["direction"] = direction
 
         self.config_db.create_entry("MIRROR_SESSION", name, mirror_entry)
 

tests/dvslib/dvs_policer.py

@@ -3,7 +3,7 @@ def __init__(self, adb, cdb):
         self.asic_db = adb
         self.config_db = cdb
 
-    def create_policer(self, name, type="packets", cir="600", cbs="600", mode="sr_tcm", red_action="drop" ):
+    def create_policer(self, name, type="packets", cir="600", cbs="600", mode="sr_tcm", red_action="drop"):
         policer_entry = {"meter_type": type, "mode": mode,
                          "cir": cir, "cbs": cbs, "red_packet_action": red_action}
         self.config_db.create_entry("POLICER", name, policer_entry)

tests/test_warm_reboot.py

@@ -2191,6 +2191,129 @@ def test_VrfMgrdWarmRestart(self, dvs, testlog):
         dvs.servers[1].runcmd("ifconfig eth0 0")
         time.sleep(2)
 
+    @pytest.fixture(scope="class")
+    def setup_erspan_neighbors(self, dvs):
+        dvs.setup_db()
+
+        dvs.set_interface_status("Ethernet12", "up")
+        dvs.set_interface_status("Ethernet16", "up")
+        dvs.set_interface_status("Ethernet20", "up")
+
+        dvs.add_ip_address("Ethernet12", "10.0.0.0/31")
+        dvs.add_ip_address("Ethernet16", "11.0.0.0/31")
+        dvs.add_ip_address("Ethernet20", "12.0.0.0/31")
+
+        dvs.add_neighbor("Ethernet12", "10.0.0.1", "02:04:06:08:10:12")
+        dvs.add_neighbor("Ethernet16", "11.0.0.1", "03:04:06:08:10:12")
+        dvs.add_neighbor("Ethernet20", "12.0.0.1", "04:04:06:08:10:12")
+
+        dvs.add_route("2.2.2.2", "10.0.0.1")
+
+        yield
+
+        dvs.remove_route("2.2.2.2")
+
+        dvs.remove_neighbor("Ethernet12", "10.0.0.1")
+        dvs.remove_neighbor("Ethernet16", "11.0.0.1")
+        dvs.remove_neighbor("Ethernet20", "12.0.0.1")
+
+        dvs.remove_ip_address("Ethernet12", "10.0.0.0/31")
+        dvs.remove_ip_address("Ethernet16", "11.0.0.0/31")
+        dvs.remove_ip_address("Ethernet20", "12.0.0.1/31")
+
+        dvs.set_interface_status("Ethernet12", "down")
+        dvs.set_interface_status("Ethernet16", "down")
+        dvs.set_interface_status("Ethernet20", "down")
+
+    @pytest.mark.usefixtures("dvs_mirror_manager", "setup_erspan_neighbors")
+    def test_MirrorSessionWarmReboot(self, dvs):
+        dvs.setup_db()
+
+        # Setup the mirror session
+        self.dvs_mirror.create_erspan_session("test_session", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0")
+
+        # Verify the monitor port
+        state_db = dvs.get_state_db()
+        state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"})
+
+        # Setup ECMP routes to the session destination
+        dvs.change_route_ecmp("2.2.2.2", ["12.0.0.1", "11.0.0.1", "10.0.0.1"])
+
+        # Monitor port should not change b/c routes are ECMP
+        state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"})
+
+        dvs.runcmd("config warm_restart enable swss")
+        dvs.stop_swss()
+        dvs.start_swss()
+
+        dvs.check_services_ready()
+        dvs.check_swss_ready()
+
+        # Monitor port should not change b/c destination is frozen
+        state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"monitor_port": "Ethernet12"})
+
+        self.dvs_mirror.remove_mirror_session("test_session")
+
+        # Revert the route back to the fixture-defined route
+        dvs.change_route("2.2.2.2", "10.0.0.1")
+
+    @pytest.mark.usefixtures("dvs_mirror_manager", "dvs_policer_manager", "setup_erspan_neighbors")
+    def test_EverflowWarmReboot(self, dvs, dvs_acl):
+        # Setup the policer
+        self.dvs_policer.create_policer("test_policer")
+        self.dvs_policer.verify_policer("test_policer")
+
+        # Setup the mirror session
+        self.dvs_mirror.create_erspan_session("test_session", "1.1.1.1", "2.2.2.2", "0x6558", "8", "100", "0", policer="test_policer")
+
+        state_db = dvs.get_state_db()
+        state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"status": "active"})
+
+        # Create the mirror table
+        dvs_acl.create_acl_table("EVERFLOW_TEST", "MIRROR", ["Ethernet12"])
+
+
+        # TODO: The standard methods for counting ACL tables and ACL rules break down after warm reboot b/c the OIDs
+        # of the default tables change. We also can't just re-initialize the default value b/c we've added another
+        # table and rule that aren't part of the base device config. We should come up with a way to handle warm reboot
+        # changes more gracefully to make it easier for future tests.
+        asic_db = dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", 1 + len(asic_db.default_acl_tables))
+
+        # Create a mirror rule
+        dvs_acl.create_mirror_acl_rule("EVERFLOW_TEST", "TEST_RULE", {"SRC_IP": "20.0.0.2"}, "test_session")
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", 1 + len(asic_db.default_acl_entries))
+
+        # Execute the warm reboot
+        dvs.runcmd("config warm_restart enable swss")
+        dvs.stop_swss()
+        dvs.start_swss()
+
+        # Make sure the system is stable
+        dvs.check_services_ready()
+        dvs.check_swss_ready()
+
+        # Verify that the ASIC DB is intact
+        self.dvs_policer.verify_policer("test_policer")
+        state_db.wait_for_field_match("MIRROR_SESSION_TABLE", "test_session", {"status": "active"})
+
+        asic_db = dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", 1 + len(asic_db.default_acl_tables))
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", 1 + len(asic_db.default_acl_entries))
+
+        # Clean up
+        dvs_acl.remove_acl_rule("EVERFLOW_TEST", "TEST_RULE")
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_ENTRY", len(asic_db.default_acl_entries))
+
+        dvs_acl.remove_acl_table("EVERFLOW_TEST")
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_ACL_TABLE", len(asic_db.default_acl_tables))
+
+        self.dvs_mirror.remove_mirror_session("test_session")
+        self.dvs_mirror.verify_no_mirror()
+
+        self.dvs_policer.remove_policer("test_policer")
+        self.dvs_policer.verify_no_policer()
+
 
 # Add Dummy always-pass test at end as workaroud
 # for issue when Flaky fail on final test it invokes module tear-down before retrying