From 03a0e21807c4998424e4a1864e6aaf651fa8cb70 Mon Sep 17 00:00:00 2001 From: Prince Sunny Date: Sun, 7 Mar 2021 12:17:09 -0800 Subject: [PATCH] [mux] VS test for neigh, route and fdb (#1656) VS test to cover: IPv4 and V6 neighbor add and simulate mux state change. Route add and simulate mux state change. ECMP route add and simulate mux state change. Neighbor handing based on FDB learnt. --- tests/test_mux.py | 405 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 343 insertions(+), 62 deletions(-) diff --git a/tests/test_mux.py b/tests/test_mux.py index 659c686e19..c1eaae75ed 100644 --- a/tests/test_mux.py +++ b/tests/test_mux.py @@ -1,5 +1,6 @@ import time import pytest +import json from swsscommon import swsscommon @@ -7,23 +8,28 @@ def create_fvs(**kwargs): return swsscommon.FieldValuePairs(list(kwargs.items())) -def create_entry(tbl, key, pairs): - fvs = swsscommon.FieldValuePairs(pairs) - tbl.set(key, fvs) - time.sleep(1) - -def create_entry_tbl(db, table, separator, key, pairs): - tbl = swsscommon.Table(db, table) - create_entry(tbl, key, pairs) - +tunnel_nh_id = 0 class TestMuxTunnelBase(object): + APP_MUX_CABLE = "MUX_CABLE_TABLE" APP_TUNNEL_DECAP_TABLE_NAME = "TUNNEL_DECAP_TABLE" ASIC_TUNNEL_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL" ASIC_TUNNEL_TERM_ENTRIES = "ASIC_STATE:SAI_OBJECT_TYPE_TUNNEL_TERM_TABLE_ENTRY" ASIC_RIF_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE" ASIC_VRF_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_VIRTUAL_ROUTER" - + ASIC_NEIGH_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY" + ASIC_NEXTHOP_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" + ASIC_ROUTE_TABLE = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" + CONFIG_MUX_CABLE = "MUX_CABLE" + + SERV1_IPV4 = "192.168.0.100" + SERV1_IPV6 = "fc02:1000::100" + SERV2_IPV4 = "192.168.0.101" + SERV2_IPV6 = "fc02:1000::101" + IPV4_MASK = "/32" + IPV6_MASK = "/128" + TUNNEL_NH_ID = 0 + ecn_modes_map = { "standard" : "SAI_TUNNEL_DECAP_ECN_MODE_STANDARD", "copy_from_outer": "SAI_TUNNEL_DECAP_ECN_MODE_COPY_FROM_OUTER" @@ -40,45 +46,299 @@ class TestMuxTunnelBase(object): } + def create_vlan_interface(self, confdb, asicdb, dvs): + + fvs = {"vlanid": "1000"} + confdb.create_entry("VLAN", "Vlan1000", fvs) + + fvs = {"tagging_mode": "untagged"} + confdb.create_entry("VLAN_MEMBER", "Vlan1000|Ethernet0", fvs) + confdb.create_entry("VLAN_MEMBER", "Vlan1000|Ethernet4", fvs) + + fvs = {"NULL": "NULL"} + confdb.create_entry("VLAN_INTERFACE", "Vlan1000", fvs) + confdb.create_entry("VLAN_INTERFACE", "Vlan1000|192.168.0.1/24", fvs) + + dvs.runcmd("config interface startup Ethernet0") + dvs.runcmd("config interface startup Ethernet4") + + + def create_mux_cable(self, confdb): + + fvs = { "server_ipv4":self.SERV1_IPV4+self.IPV4_MASK, "server_ipv6":self.SERV1_IPV6+self.IPV6_MASK } + confdb.create_entry(self.CONFIG_MUX_CABLE, "Ethernet0", fvs) + + fvs = { "server_ipv4":self.SERV2_IPV4+self.IPV4_MASK, "server_ipv6":self.SERV2_IPV6+self.IPV6_MASK } + confdb.create_entry(self.CONFIG_MUX_CABLE, "Ethernet4", fvs) + + + def set_mux_state(self, appdb, ifname, state_change): + + ps = swsscommon.ProducerStateTable(appdb, self.APP_MUX_CABLE) + + fvs = create_fvs(state=state_change) + + ps.set(ifname, fvs) + + time.sleep(1) + + + def check_neigh_in_asic_db(self, asicdb, ip, expected=1): + + nbr = asicdb.wait_for_n_keys(self.ASIC_NEIGH_TABLE, expected) + + found = False + for key in nbr: + entry = json.loads(key) + if entry["ip"] == ip: + found = True + entry = key + break + + assert found + return entry + + + def check_tnl_nexthop_in_asic_db(self, asicdb, expected=1): + + global tunnel_nh_id + + nh = asicdb.wait_for_n_keys(self.ASIC_NEXTHOP_TABLE, expected) + + for key in nh: + fvs = asicdb.get_entry(self.ASIC_NEXTHOP_TABLE, key) + if fvs.get("SAI_NEXT_HOP_ATTR_TYPE") == "SAI_NEXT_HOP_TYPE_TUNNEL_ENCAP": + tunnel_nh_id = key + + assert tunnel_nh_id + + + def check_nexthop_in_asic_db(self, asicdb, key, standby=False): + + fvs = asicdb.get_entry(self.ASIC_ROUTE_TABLE, key) + if not fvs: + assert False + + nhid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") + if standby: + assert (nhid == tunnel_nh_id) + else: + assert (nhid != tunnel_nh_id) + + + def check_nexthop_group_in_asic_db(self, asicdb, key, num_tnl_nh=0): + + fvs = asicdb.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY", key) + + nhg_id = fvs["SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID"] + + asicdb.wait_for_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhg_id) + + # Two NH group members are expected to be added + keys = asicdb.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", 2) + + count = 0 + + for k in keys: + fvs = asicdb.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER", k) + assert fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID"] == nhg_id + + # Count the number of Nexthop member pointing to tunnel + if fvs["SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID"] == tunnel_nh_id: + count += 1 + + assert num_tnl_nh == count + + + def add_neighbor(self, dvs, ip, mac, v6=False): + + if v6: + dvs.runcmd("ip -6 neigh replace " + ip + " lladdr " + mac + " dev Vlan1000") + else: + dvs.runcmd("ip -4 neigh replace " + ip + " lladdr " + mac + " dev Vlan1000") + + + def add_fdb(self, dvs, port, mac): + + appdb = dvs.get_app_db() + ps = swsscommon.ProducerStateTable(appdb.db_connection, "FDB_TABLE") + fvs = swsscommon.FieldValuePairs([("port", port), ("type", "dynamic")]) + + ps.set("Vlan1000:"+mac, fvs) + + time.sleep(1) + + + def create_and_test_neighbor(self, confdb, appdb, asicdb, dvs, dvs_route): + + self.create_vlan_interface(confdb, asicdb, dvs) + + self.create_mux_cable(confdb) + + self.set_mux_state(appdb, "Ethernet0", "active") + self.set_mux_state(appdb, "Ethernet4", "standby") + + self.add_neighbor(dvs, self.SERV1_IPV4, "00:00:00:00:00:01") + # Broadcast neigh 192.168.0.255 is default added. Hence +1 for expected number + srv1_v4 = self.check_neigh_in_asic_db(asicdb, self.SERV1_IPV4, 2) + + self.add_neighbor(dvs, self.SERV1_IPV6, "00:00:00:00:00:01", True) + srv1_v6 = self.check_neigh_in_asic_db(asicdb, self.SERV1_IPV6, 3) + + existing_keys = asicdb.get_keys(self.ASIC_NEIGH_TABLE) + + self.add_neighbor(dvs, self.SERV2_IPV4, "00:00:00:00:00:02") + self.add_neighbor(dvs, self.SERV2_IPV6, "00:00:00:00:00:02", True) + time.sleep(1) + + # In standby mode, the entry must not be added to Neigh table but Route + asicdb.wait_for_matching_keys(self.ASIC_NEIGH_TABLE, existing_keys) + dvs_route.check_asicdb_route_entries([self.SERV2_IPV4+self.IPV4_MASK, self.SERV2_IPV6+self.IPV6_MASK]) + + # The first standby route also creates as tunnel Nexthop + self.check_tnl_nexthop_in_asic_db(asicdb, 3) + + # Change state to Standby. This will delete Neigh and add Route + self.set_mux_state(appdb, "Ethernet0", "standby") + + asicdb.wait_for_deleted_entry(self.ASIC_NEIGH_TABLE, srv1_v4) + asicdb.wait_for_deleted_entry(self.ASIC_NEIGH_TABLE, srv1_v6) + dvs_route.check_asicdb_route_entries([self.SERV1_IPV4+self.IPV4_MASK, self.SERV1_IPV6+self.IPV6_MASK]) + + # Change state to Active. This will add Neigh and delete Route + self.set_mux_state(appdb, "Ethernet4", "active") + + dvs_route.check_asicdb_deleted_route_entries([self.SERV2_IPV4+self.IPV4_MASK, self.SERV2_IPV6+self.IPV6_MASK]) + self.check_neigh_in_asic_db(asicdb, self.SERV2_IPV4, 3) + self.check_neigh_in_asic_db(asicdb, self.SERV2_IPV6, 3) + + + def create_and_test_fdb(self, appdb, asicdb, dvs, dvs_route): + + self.set_mux_state(appdb, "Ethernet0", "active") + self.set_mux_state(appdb, "Ethernet4", "standby") + + self.add_fdb(dvs, "Ethernet0", "00-00-00-00-00-11") + self.add_fdb(dvs, "Ethernet4", "00-00-00-00-00-12") + + ip_1 = "fc02:1000::10" + ip_2 = "fc02:1000::11" + + self.add_neighbor(dvs, ip_1, "00:00:00:00:00:11", True) + self.add_neighbor(dvs, ip_2, "00:00:00:00:00:12", True) + + # ip_1 is on Active Mux, hence added to Host table + self.check_neigh_in_asic_db(asicdb, ip_1, 4) + + # ip_2 is on Standby Mux, hence added to Route table + dvs_route.check_asicdb_route_entries([ip_2+self.IPV6_MASK]) + + # Check ip_1 move to standby mux, should be pointing to tunnel + self.add_neighbor(dvs, ip_1, "00:00:00:00:00:12", True) + + # ip_1 moved to standby Mux, hence added to Route table + dvs_route.check_asicdb_route_entries([ip_1+self.IPV6_MASK]) + + # Check ip_2 move to active mux, should be host entry + self.add_neighbor(dvs, ip_2, "00:00:00:00:00:11", True) + + # ip_2 moved to active Mux, hence remove from Route table + dvs_route.check_asicdb_deleted_route_entries([ip_2+self.IPV6_MASK]) + self.check_neigh_in_asic_db(asicdb, ip_2, 4) + + + def create_and_test_route(self, appdb, asicdb, dvs, dvs_route): + + self.set_mux_state(appdb, "Ethernet0", "active") + + rtprefix = "2.3.4.0/24" + + dvs.runcmd("vtysh -c \"configure terminal\" -c \"ip route " + rtprefix + " " + self.SERV1_IPV4 + "\"") + + pdb = dvs.get_app_db() + pdb.wait_for_entry("ROUTE_TABLE", rtprefix) + + rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) + + self.check_nexthop_in_asic_db(asicdb, rtkeys[0]) + + # Change Mux state to Standby and verify route pointing to Tunnel + self.set_mux_state(appdb, "Ethernet0", "standby") + + self.check_nexthop_in_asic_db(asicdb, rtkeys[0], True) + + # Change Mux state back to Active and verify route is not pointing to Tunnel + self.set_mux_state(appdb, "Ethernet0", "active") + + self.check_nexthop_in_asic_db(asicdb, rtkeys[0]) + + # Test ECMP routes + + self.set_mux_state(appdb, "Ethernet0", "active") + self.set_mux_state(appdb, "Ethernet4", "active") + + rtprefix = "5.6.7.0/24" + + dvs_route.check_asicdb_deleted_route_entries([rtprefix]) + + ps = swsscommon.ProducerStateTable(pdb.db_connection, "ROUTE_TABLE") + + fvs = swsscommon.FieldValuePairs([("nexthop", self.SERV1_IPV4 + "," + self.SERV2_IPV4), ("ifname", "Vlan1000,Vlan1000")]) + + ps.set(rtprefix, fvs) + + # Check if route was propagated to ASIC DB + rtkeys = dvs_route.check_asicdb_route_entries([rtprefix]) + + # Check for nexthop group and validate nexthop group member in asic db + self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0]) + + # Step: 1 - Change one NH to standby and verify ecmp route + self.set_mux_state(appdb, "Ethernet0", "standby") + self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) + + # Step: 2 - Change the other NH to standby and verify ecmp route + self.set_mux_state(appdb, "Ethernet4", "standby") + self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 2) + + # Step: 3 - Change one NH to back to Active and verify ecmp route + self.set_mux_state(appdb, "Ethernet0", "active") + self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0], 1) + + # Step: 4 - Change the other NH to Active and verify ecmp route + self.set_mux_state(appdb, "Ethernet4", "active") + self.check_nexthop_group_in_asic_db(asicdb, rtkeys[0]) + + def check_interface_exists_in_asicdb(self, asicdb, sai_oid): - if_table = swsscommon.Table(asicdb, self.ASIC_RIF_TABLE) - status, fvs = if_table.get(sai_oid) - return status + asicdb.wait_for_entry(self.ASIC_RIF_TABLE, sai_oid) + return True + def check_vr_exists_in_asicdb(self, asicdb, sai_oid): - vfr_table = swsscommon.Table(asicdb, self.ASIC_VRF_TABLE) - status, fvs = vfr_table.get(sai_oid) - return status + asicdb.wait_for_entry(self.ASIC_VRF_TABLE, sai_oid) + return True + def create_and_test_peer(self, db, asicdb, peer_name, peer_ip, src_ip): """ Create PEER entry verify all needed enties in ASIC DB exists """ - create_entry_tbl( - db, - "PEER_SWITCH", '|', "%s" % (peer_name), - [ - ("address_ipv4", peer_ip), - ] - ) + peer_attrs = { + "address_ipv4": peer_ip + } - time.sleep(2) + db.create_entry("PEER_SWITCH", peer_name, peer_attrs) # check asic db table - tunnel_table = swsscommon.Table(asicdb, self.ASIC_TUNNEL_TABLE) - - tunnels = tunnel_table.getKeys() - # There will be two tunnels, one P2MP and another P2P - assert len(tunnels) == 2 + tunnels = asicdb.wait_for_n_keys(self.ASIC_TUNNEL_TABLE, 2) p2p_obj = None for tunnel_sai_obj in tunnels: - status, fvs = tunnel_table.get(tunnel_sai_obj) + fvs = asicdb.wait_for_entry(self.ASIC_TUNNEL_TABLE, tunnel_sai_obj) - assert status == True - - for field, value in fvs: + for field, value in fvs.items(): if field == "SAI_TUNNEL_ATTR_TYPE": assert value == "SAI_TUNNEL_TYPE_IPINIP" if field == "SAI_TUNNEL_ATTR_PEER_MODE": @@ -87,11 +347,9 @@ def create_and_test_peer(self, db, asicdb, peer_name, peer_ip, src_ip): assert p2p_obj != None - status, fvs = tunnel_table.get(p2p_obj) - - assert status == True + fvs = asicdb.wait_for_entry(self.ASIC_TUNNEL_TABLE, p2p_obj) - for field, value in fvs: + for field, value in fvs.items(): if field == "SAI_TUNNEL_ATTR_TYPE": assert value == "SAI_TUNNEL_TYPE_IPINIP" elif field == "SAI_TUNNEL_ATTR_ENCAP_SRC_IP": @@ -111,18 +369,14 @@ def create_and_test_peer(self, db, asicdb, peer_name, peer_ip, src_ip): def check_tunnel_termination_entry_exists_in_asicdb(self, asicdb, tunnel_sai_oid, dst_ips): - tunnel_term_table = swsscommon.Table(asicdb, self.ASIC_TUNNEL_TERM_ENTRIES) - - tunnel_term_entries = tunnel_term_table.getKeys() - assert len(tunnel_term_entries) == len(dst_ips) + tunnel_term_entries = asicdb.wait_for_n_keys(self.ASIC_TUNNEL_TERM_ENTRIES, len(dst_ips)) for term_entry in tunnel_term_entries: - status, fvs = tunnel_term_table.get(term_entry) + fvs = asicdb.get_entry(self.ASIC_TUNNEL_TERM_ENTRIES, term_entry) - assert status == True assert len(fvs) == 5 - for field, value in fvs: + for field, value in fvs.items(): if field == "SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_VR_ID": assert self.check_vr_exists_in_asicdb(asicdb, value) elif field == "SAI_TUNNEL_TERM_TABLE_ENTRY_ATTR_TYPE": @@ -136,6 +390,7 @@ def check_tunnel_termination_entry_exists_in_asicdb(self, asicdb, tunnel_sai_oid else: assert False, "Field %s is not tested" % field + def create_and_test_tunnel(self, db, asicdb, tunnel_name, **kwargs): """ Create tunnel and verify all needed enties in ASIC DB exists """ @@ -152,16 +407,12 @@ def create_and_test_tunnel(self, db, asicdb, tunnel_name, **kwargs): time.sleep(1) # check asic db table - tunnel_table = swsscommon.Table(asicdb, self.ASIC_TUNNEL_TABLE) - - tunnels = tunnel_table.getKeys() - assert len(tunnels) == 1 + tunnels = asicdb.wait_for_n_keys(self.ASIC_TUNNEL_TABLE, 1) tunnel_sai_obj = tunnels[0] - status, fvs = tunnel_table.get(tunnel_sai_obj) + fvs = asicdb.wait_for_entry(self.ASIC_TUNNEL_TABLE, tunnel_sai_obj) - assert status == True # 6 parameters to check in case of decap tunnel # + 1 (SAI_TUNNEL_ATTR_ENCAP_SRC_IP) in case of symmetric tunnel assert len(fvs) == 7 if is_symmetric_tunnel else 6 @@ -170,7 +421,7 @@ def create_and_test_tunnel(self, db, asicdb, tunnel_name, **kwargs): expected_dscp_mode = self.dscp_modes_map[kwargs["dscp_mode"]] expected_ttl_mode = self.ttl_modes_map[kwargs["ttl_mode"]] - for field, value in fvs: + for field, value in fvs.items(): if field == "SAI_TUNNEL_ATTR_TYPE": assert value == "SAI_TUNNEL_TYPE_IPINIP" elif field == "SAI_TUNNEL_ATTR_ENCAP_SRC_IP": @@ -190,6 +441,7 @@ def create_and_test_tunnel(self, db, asicdb, tunnel_name, **kwargs): self.check_tunnel_termination_entry_exists_in_asicdb(asicdb, tunnel_sai_obj, kwargs["dst_ip"].split(",")) + def remove_and_test_tunnel(self, db, asicdb, tunnel_name): """ Removes tunnel and checks that ASIC db is clear""" @@ -216,18 +468,19 @@ def remove_and_test_tunnel(self, db, asicdb, tunnel_name): assert len(tunnel_app_table.getKeys()) == 0 assert not self.check_interface_exists_in_asicdb(asicdb, overlay_infs_id) + def cleanup_left_over(self, db, asicdb): """ Cleanup APP and ASIC tables """ - tunnel_table = swsscommon.Table(asicdb, self.ASIC_TUNNEL_TABLE) - for key in tunnel_table.getKeys(): - tunnel_table._del(key) + tunnel_table = asicdb.get_keys(self.ASIC_TUNNEL_TABLE) + for key in tunnel_table: + asicdb.delete_entry(self.ASIC_TUNNEL_TABLE, key) - tunnel_term_table = swsscommon.Table(asicdb, self.ASIC_TUNNEL_TERM_ENTRIES) - for key in tunnel_term_table.getKeys(): - tunnel_term_table._del(key) + tunnel_term_table = asicdb.get_keys(self.ASIC_TUNNEL_TERM_ENTRIES) + for key in tunnel_term_table: + asicdb.delete_entry(self.ASIC_TUNNEL_TERM_ENTRIES, key) - tunnel_app_table = swsscommon.Table(asicdb, self.APP_TUNNEL_DECAP_TABLE_NAME) + tunnel_app_table = swsscommon.Table(db, self.APP_TUNNEL_DECAP_TABLE_NAME) for key in tunnel_app_table.getKeys(): tunnel_table._del(key) @@ -239,9 +492,9 @@ def test_Tunnel(self, dvs, testlog): """ test IPv4 Mux tunnel creation """ db = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) - asicdb = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + asicdb = dvs.get_asic_db() - self.cleanup_left_over(db, asicdb) + #self.cleanup_left_over(db, asicdb) # create tunnel IPv4 tunnel self.create_and_test_tunnel(db, asicdb, tunnel_name="MuxTunnel0", tunnel_type="IPINIP", @@ -252,12 +505,40 @@ def test_Tunnel(self, dvs, testlog): def test_Peer(self, dvs, testlog): """ test IPv4 Mux tunnel creation """ - db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) - asicdb = swsscommon.DBConnector(swsscommon.ASIC_DB, dvs.redis_sock, 0) + db = dvs.get_config_db() + asicdb = dvs.get_asic_db() self.create_and_test_peer(db, asicdb, "peer", "1.1.1.1", "10.1.0.32") + def test_Neighbor(self, dvs, dvs_route, testlog): + """ test Neighbor entries and mux state change """ + + confdb = dvs.get_config_db() + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + asicdb = dvs.get_asic_db() + + self.create_and_test_neighbor(confdb, appdb, asicdb, dvs, dvs_route) + + + def test_Fdb(self, dvs, dvs_route, testlog): + """ test Fdb entries and mux state change """ + + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + asicdb = dvs.get_asic_db() + + self.create_and_test_fdb(appdb, asicdb, dvs, dvs_route) + + + def test_Route(self, dvs, dvs_route, testlog): + """ test Route entries and mux state change """ + + appdb = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + asicdb = dvs.get_asic_db() + + self.create_and_test_route(appdb, asicdb, dvs, dvs_route) + + # Add Dummy always-pass test at end as workaroud # for issue when Flaky fail on final test it invokes module tear-down before retrying def test_nonflaky_dummy():