From 7fa7cd6eba07106a9387c437344873a9940cce4d Mon Sep 17 00:00:00 2001 From: lguohan Date: Sat, 31 Oct 2020 22:58:48 -0700 Subject: [PATCH] [vstest]: stablize fgnhg test (#1491) use standard wait API to wait for asic db and state db updates Signed-off-by: Guohan Lu --- tests/test_fgnhg.py | 59 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/tests/test_fgnhg.py b/tests/test_fgnhg.py index e273484541bf..3baf1bb28bb6 100644 --- a/tests/test_fgnhg.py +++ b/tests/test_fgnhg.py @@ -111,7 +111,7 @@ def _access_function(): elif key == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": if nhgid != val: return false_ret - if (index == -1 or + if (index == -1 or nh_oid == "0" or nh_oid_map.get(nh_oid,"NULL") == "NULL" or nh_oid_map.get(nh_oid) not in nh_memb_exp_count): @@ -149,18 +149,18 @@ def verify_programmed_fg_state_db_entry(state_db,nh_memb_exp_count): memb_dict = nh_memb_exp_count keys = state_db.get_keys("FG_ROUTE_TABLE") assert len(keys) != 0 - for key in keys: + for key in keys: fvs = state_db.get_entry("FG_ROUTE_TABLE", key) assert fvs != {} for key, value in fvs.items(): assert value in nh_memb_exp_count - memb_dict[value] = memb_dict[value] - 1 + memb_dict[value] = memb_dict[value] - 1 for idx,memb in memb_dict.items(): - assert memb == 0 + assert memb == 0 -def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, +def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size): state_db_entry_memb_exp_count = {} @@ -227,11 +227,11 @@ def test_route_fgnhg(self, dvs, testlog): asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB)) ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), ("ifname", "Ethernet12,Ethernet16,Ethernet20")]) ps.set(fg_nhg_prefix, fvs) - # No ASIC_DB entry we can wait for since ARP is not resolved yet, + # No ASIC_DB entry we can wait for since ARP is not resolved yet, # We just use sleep so that the sw receives this entry time.sleep(1) @@ -261,7 +261,7 @@ def test_route_fgnhg(self, dvs, testlog): validate_asic_nhg(asic_db, nhgid, bucket_size) - nh_oid_map = get_nh_oid_map(asic_db) + nh_oid_map = get_nh_oid_map(asic_db) ### Test scenarios with bank 0 having 0 members up # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7 @@ -272,7 +272,7 @@ def test_route_fgnhg(self, dvs, testlog): # Resolve ARP for 10.0.0.7 asic_nh_count = len(asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP")) dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04") - asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", asic_nh_count + 1) + asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", asic_nh_count + 1) nh_oid_map = get_nh_oid_map(asic_db) # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} @@ -359,13 +359,13 @@ def test_route_fgnhg(self, dvs, testlog): nh_memb_exp_count = {"10.0.0.11":60} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - + # Bring down last link, there shouldn't be a crash or other bad orchagent state because of this shutdown_link(dvs, app_db, 5) # Nothing to check for in this case, sleep 1s for the shutdown to reach sw time.sleep(1) - # bring all links up one by one + # bring all links up one by one startup_link(dvs, app_db, 3) startup_link(dvs, app_db, 4) startup_link(dvs, app_db, 5) @@ -399,14 +399,12 @@ def test_route_fgnhg(self, dvs, testlog): # Remove route asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) ps._del(fg_nhg_prefix) - asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) - keys = asic_db.get_keys(ASIC_NHG_MEMB) - assert len(keys) == 0 + # validate routes and nhg member in asic db, route entry in state db are removed + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) + state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0) - keys = state_db.get_keys("FG_ROUTE_TABLE") - assert len(keys) == 0 - remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix) # Nothing we can wait for in terms of db entries, we sleep here # to give the sw enough time to delete the entry @@ -415,7 +413,7 @@ def test_route_fgnhg(self, dvs, testlog): # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see # standard(non-Fine grained) ECMP behavior asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB)) - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), ("ifname", "Ethernet12,Ethernet16,Ethernet20")]) ps.set(fg_nhg_prefix, fvs) @@ -423,9 +421,8 @@ def test_route_fgnhg(self, dvs, testlog): nhgid = asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix) assert nhgid is not None - keys = asic_db.get_keys(ASIC_NHG_MEMB) - assert len(keys) == 3 - + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3) + # add fgnhg prefix: The regular route should transition to fine grained ECMP fvs = {"FG_NHG": fg_nhg_name} create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs) @@ -435,10 +432,10 @@ def test_route_fgnhg(self, dvs, testlog): keys = asic_db.get_keys(ASIC_ROUTE_TB) nhgid = asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix) validate_asic_nhg(asic_db, nhgid, bucket_size) - + nh_oid_map = {} - nh_oid_map = get_nh_oid_map(asic_db) - + nh_oid_map = get_nh_oid_map(asic_db) + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) @@ -449,7 +446,7 @@ def test_route_fgnhg(self, dvs, testlog): # Validate regular ECMP asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3) asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix) - + # remove prefix entry asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) ps._del(fg_nhg_prefix) @@ -458,7 +455,7 @@ def test_route_fgnhg(self, dvs, testlog): # Cleanup all FG, arp and interface remove_entry(config_db, "FG_NHG", fg_nhg_name) - + for i in range(0,NUM_NHs): if_name_key = "Ethernet" + str(i*4) ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" @@ -469,8 +466,8 @@ def test_route_fgnhg(self, dvs, testlog): remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2)) - ### Create new set of entries with a greater number of FG members and - ### bigger bucket size such that the # of nhs are not divisible by + ### Create new set of entries with a greater number of FG members and + ### bigger bucket size such that the # of nhs are not divisible by ### bucket size. fg_nhg_name = "new_fgnhg_v4" fg_nhg_prefix = "3.3.3.0/24" @@ -503,10 +500,10 @@ def test_route_fgnhg(self, dvs, testlog): create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key dvs.runcmd("arp -s 10.0.0." + str(1 + i*2) + " 00:00:00:00:00:" + str(1 + i*2)) - + # Program the route asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB)) - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.11"), + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.11"), ("ifname", "Ethernet0,Ethernet20")]) ps.set(fg_nhg_prefix, fvs) @@ -517,7 +514,7 @@ def test_route_fgnhg(self, dvs, testlog): validate_asic_nhg(asic_db, nhgid, bucket_size) - nh_oid_map = get_nh_oid_map(asic_db) + nh_oid_map = get_nh_oid_map(asic_db) # Test addition of route with 0 members in bank nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.11":64}