Test case 2 of PFC watchdog against warm-reboot: sad path (#834)

* First test case of PFC watchdog against warm-reboot Signed-off-by: Wenda Ni <[email protected]> * Add more comments for code readability Signed-off-by: Wenda Ni <[email protected]> * First test case of PFC watchdog against warm-reboot Signed-off-by: Wenda Ni <[email protected]> * Add more comments for code readability Signed-off-by: Wenda Ni <[email protected]> * Modify output message Signed-off-by: Wenda Ni <[email protected]> * Allow log analyzer to take a specified start marker Signed-off-by: Wenda Ni <[email protected]> * Use lookup('pipe', 'date +%H:%M:%S') in place of ansible_date_time.time, which uses cached time for a certain period of time ansible/ansible#22561 Signed-off-by: Wenda Ni <[email protected]> * Add the flexiblity to not start storm at fanout link partener in running functional_test_storm.yml Signed-off-by: Wenda Ni <[email protected]> * Dump only the current result and summary files for debugging and troubleshooting purpose Signed-off-by: Wenda Ni <[email protected]> * Add the capability to check if the number of exact matches is equal to to the target number Signed-off-by: Wenda Ni <[email protected]> * Split the actual storm and restore tests into functional_test_storm_perq.yml and functional_test_restore_perq.yml, respectively Add the capability to storm multiple queues of a port Signed-off-by: Wenda Ni <[email protected]> * Add test case 2 of PFC watchdog against warm-reboot: PFC storm started and detected before warm-reboot On-going storm on warm-reboot emission, and lasts past the warm-reboot finish PFC storm stopped and restored after warm-reboot Signed-off-by: Wenda Ni <[email protected]> * Ignore trival syncd ERR during the warm-reboot, e.g., Mar 20 00:40:33.599212 str-a7050-acs-1 ERR syncd#syncd: _brcm_sai_cosq_stat_get:1146 cosq stat get failed with error Invalid parameter (0xfffffffc). Mar 20 00:40:33.599212 str-a7050-acs-1 DEBUG syncd#syncd: brcm_sai_get_queue_stats:724 cosq stat get failed with error -5 for port 1 qid 10 Mar 20 00:40:33.599212 str-a7050-acs-1 NOTICE syncd#syncd: :- setQueueCounterList: Queue oid:0x102150000000b does not has supported counters Signed-off-by: Wenda Ni <[email protected]> * Use boolean variable to determine the test run type: regular pfc wd test or pfcwd warm-reboot test Signed-off-by: Wenda Ni <[email protected]> * Feed reboot type to reboot_sonic.yml in warm-reboot happy path test Signed-off-by: Wenda Ni <[email protected]> * Feed reboot type to reboot_sonic.yml in warm-reboot sad path test Signed-off-by: Wenda Ni <[email protected]> * Add expected errors on mlnx platform Signed-off-by: Wenda Ni <[email protected]>
sonic-net · Oct 15, 2019 · f523ec5 · f523ec5
1 parent 4de6e7a
commit f523ec5
Show file tree

Hide file tree

Showing 6 changed files with 235 additions and 167 deletions.
diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml
@@ -1,5 +1,6 @@
 #------------------------------------
 # Test the PFC restore action
+# Iterate the test over a list of lossless queues
 # 1. Verify restore is detected via syslog entry analysis on the dut
 # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
 #------------------------------------
@@ -49,16 +50,18 @@
 
 - name: Prepare variables required for PFC test
   set_fact:
-    pfc_queue_index: 4
+    pfc_queue_indices: [4]
     pfc_frames_number: 100000000
     pfc_wd_test_pkt_count: 100
     pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}"
     peer_hwsku: "{{device_info['HwSku']}}"
     peer_mgmt: "{{device_info['mgmtip']}}"
     testname: functional_test
 
-- set_fact:
-    class_enable: "{{(1).__lshift__(pfc_queue_index)}}"
+- name: Add queue index 3 to pfc_queue_indices when seed is an odd number
+  set_fact:
+    pfc_queue_indices: "{{pfc_queue_indices + [3]}}"
+  when: seed | int is odd
 
 - set_fact:
       peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
@@ -67,76 +70,9 @@
   include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml
 
 
-- block:
-    # 1. Verify restore is detected via syslog entry analysis on the dut
-    - set_fact:
-        test_expect_file: "expect_pfc_wd_restore"
-
-    - name: Initialize loganalyzer
-      include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml
-
-    - name: Stop PFC storm on fanout switch
-      action: apswitch template="{{pfc_wd_storm_stop_template}}"
-      args:
-        host: "{{peer_mgmt}}"
-        login: "{{peer_login}}"
-      connection: switch
-
-    - name: Wait for queue to recover from PFC storm
-      pause:
-        seconds: 1
-
-    - name: Check if logs contain message that PFC WD restored from deadlock
-      include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml
-
-    - name: Check if logs contain message that PFC WD restored from deadlock
-      include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml
-
-    # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
-    - name: "Send packets via {{pfc_wd_test_port}}"
-      include: roles/test/tasks/ptf_runner.yml
-      vars:
-        ptf_test_name: PFC WD test
-        ptf_test_dir: ptftests
-        ptf_test_path: pfc_wd.PfcWdTest
-        ptf_platform: remote
-        ptf_platform_dir: ptftests
-        ptf_test_params:
-          - testbed_type='{{testbed_type}}'
-          - router_mac='{{ansible_ethernet0_mac_addr}}'
-          - queue_index='{{pfc_queue_index}}'
-          - pkt_count='{{pfc_wd_test_pkt_count}}'
-          - port_src='{{pfc_wd_rx_port_id[0]}}'
-          - port_dst='{{pfc_wd_test_port_ids}}'
-          - ip_dst='{{pfc_wd_test_neighbor_addr}}'
-          - port_type='{{port_type}}'
-          - wd_action='forward'
-        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
-
-    - name: "Send packets to {{pfc_wd_test_port}}"
-      include: roles/test/tasks/ptf_runner.yml
-      vars:
-        ptf_test_name: PFC WD test
-        ptf_test_dir: ptftests
-        ptf_test_path: pfc_wd.PfcWdTest
-        ptf_platform: remote
-        ptf_platform_dir: ptftests
-        ptf_test_params:
-          - testbed_type='{{testbed_type}}'
-          - router_mac='{{ansible_ethernet0_mac_addr}}'
-          - queue_index='{{pfc_queue_index}}'
-          - pkt_count='{{pfc_wd_test_pkt_count}}'
-          - port_src='{{pfc_wd_test_port_id}}'
-          - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]'
-          - ip_dst='{{pfc_wd_rx_neighbor_addr}}'
-          - port_type='{{port_type}}'
-          - wd_action='forward'
-        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
-
-  rescue:
-    - name: Stop PFC storm on fanout switch
-      action: apswitch template="{{pfc_wd_storm_stop_template}}"
-      args:
-        host: "{{peer_mgmt}}"
-        login: "{{peer_login}}"
-      connection: switch
+- name: Test PFC restore function per queue
+  include: roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml
+  vars:
+    pfc_queue_index: "{{item}}"
+    class_enable: "{{(1).__lshift__(item)}}"
+  with_items: "{{pfc_queue_indices}}"
diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml
@@ -0,0 +1,81 @@
+#------------------------------------
+# Test the PFC restore action
+# 1. Verify restore is detected via syslog entry analysis on the dut
+# 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+#------------------------------------
+
+- block:
+    # 1. Verify restore is detected via syslog entry analysis on the dut
+    - set_fact:
+        test_expect_file: "expect_pfc_wd_restore"
+
+    - name: Initialize loganalyzer
+      include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml
+      vars:
+        testname_unique_gen: true
+
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch
+
+    - name: Allow enough time for the PFC storm restoration to flow into the syslog
+      pause:
+        seconds: 1
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml
+
+    - name: Check if logs contain message that PFC WD restored from deadlock
+      include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml
+
+    # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test
+    - name: "Send packets via {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_rx_port_id[0]}}'
+          - port_dst='{{pfc_wd_test_port_ids}}'
+          - ip_dst='{{pfc_wd_test_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+    - name: "Send packets to {{pfc_wd_test_port}}"
+      include: roles/test/tasks/ptf_runner.yml
+      vars:
+        ptf_test_name: PFC WD test
+        ptf_test_dir: ptftests
+        ptf_test_path: pfc_wd.PfcWdTest
+        ptf_platform: remote
+        ptf_platform_dir: ptftests
+        ptf_test_params:
+          - testbed_type='{{testbed_type}}'
+          - router_mac='{{ansible_ethernet0_mac_addr}}'
+          - queue_index='{{pfc_queue_index}}'
+          - pkt_count='{{pfc_wd_test_pkt_count}}'
+          - port_src='{{pfc_wd_test_port_id}}'
+          - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]'
+          - ip_dst='{{pfc_wd_rx_neighbor_addr}}'
+          - port_type='{{port_type}}'
+          - wd_action='forward'
+        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
+
+  rescue:
+    - name: Stop PFC storm on fanout switch
+      action: apswitch template="{{pfc_wd_storm_stop_template}}"
+      args:
+        host: "{{peer_mgmt}}"
+        login: "{{peer_login}}"
+      connection: switch
diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml
@@ -1,5 +1,6 @@
 #------------------------------------
-# Test the PFC storm action
+# Test the PFC storm action on a port
+# Iterate the test over a list of lossless queues
 # 1. Verify storm is detected via syslog entry analysis on the dut
 # 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test
 #
@@ -51,16 +52,18 @@
 
 - name: Prepare variables required for PFC test
   set_fact:
-    pfc_queue_index: 4
+    pfc_queue_indices: [4]
     pfc_frames_number: 100000000
     pfc_wd_test_pkt_count: 100
     pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}"
     peer_hwsku: "{{device_info['HwSku']}}"
     peer_mgmt: "{{device_info['mgmtip']}}"
     testname: functional_test
 
-- set_fact:
-    class_enable: "{{(1).__lshift__(pfc_queue_index)}}"
+- name: Add queue index 3 to pfc_queue_indices when seed is an odd number
+  set_fact:
+    pfc_queue_indices: "{{pfc_queue_indices + [3]}}"
+  when: seed | int is odd
 
 - set_fact:
       peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
@@ -69,90 +72,10 @@
   include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml
 
 
-- block:
-    - set_fact:
-          pfc_gen_file: pfc_gen.py
-
-    - name: Deploy pfc packet generater file to fanout switch
-      include: roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml
-
-    - name: copy the test to ptf container
-      copy: src=roles/test/files/ptftests dest=/root
-      delegate_to: "{{ptf_host}}"
-
-    # 1. Verify storm is detected via syslog entry analysis on the dut
-    - set_fact:
-        test_expect_file: "expect_pfc_wd_detect"
-        test_ignore_file: "ignore_pfc_wd_messages"
-
-    - name: Initialize loganalyzer
-      include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml
-
-    - name: Generate PFC storm on fanout switch
-      action: apswitch template="{{pfc_wd_storm_template}}"
-      args:
-        host: "{{peer_mgmt}}"
-        login: "{{peer_login}}"
-      connection: switch
-      register: pid
-
-    - debug: msg="{{pid}}"
-
-    - name: Let PFC storm happen for a while
-      pause:
-        seconds: 5
-
-    - name: Check if logs contain message that PFC WD detected storm
-      include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml
-
-    - name: Check if logs contain message that PFC WD detected storm
-      include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml
-
-    # 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test
-    - name: "check egress drop, tx port {{pfc_wd_test_port}}"
-      include: roles/test/tasks/ptf_runner.yml
-      vars:
-        ptf_test_name: PFC WD test
-        ptf_test_dir: ptftests
-        ptf_test_path: pfc_wd.PfcWdTest
-        ptf_platform: remote
-        ptf_platform_dir: ptftests
-        ptf_test_params:
-          - testbed_type='{{testbed_type}}'
-          - router_mac='{{ansible_ethernet0_mac_addr}}'
-          - queue_index='{{pfc_queue_index}}'
-          - pkt_count='{{pfc_wd_test_pkt_count}}'
-          - port_src='{{pfc_wd_rx_port_id[0]}}'
-          - port_dst='[{{pfc_wd_test_port_id}}]'
-          - ip_dst='{{pfc_wd_test_neighbor_addr}}'
-          - port_type='{{port_type}}'
-          - wd_action='drop'
-        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
-
-    - name: "Check ingress drop, rx port {{pfc_wd_test_port}}"
-      include: roles/test/tasks/ptf_runner.yml
-      vars:
-        ptf_test_name: PFC WD test
-        ptf_test_dir: ptftests
-        ptf_test_path: pfc_wd.PfcWdTest
-        ptf_platform: remote
-        ptf_platform_dir: ptftests
-        ptf_test_params:
-          - testbed_type='{{testbed_type}}'
-          - router_mac='{{ansible_ethernet0_mac_addr}}'
-          - queue_index='{{pfc_queue_index}}'
-          - pkt_count='{{pfc_wd_test_pkt_count}}'
-          - port_src='{{pfc_wd_test_port_id}}'
-          - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]'
-          - ip_dst='{{pfc_wd_rx_neighbor_addr}}'
-          - port_type='{{port_type}}'
-          - wd_action='drop'
-        ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log "
-
-  rescue:
-    - name: Stop PFC storm on fanout switch
-      action: apswitch template="{{pfc_wd_storm_stop_template}}"
-      args:
-        host: "{{peer_mgmt}}"
-        login: "{{peer_login}}"
-      connection: switch
+- name: Test PFC storm function per queue
+  include: roles/test/tasks/pfc_wd/functional_test/functional_test_storm_perq.yml
+  vars:
+    pfc_queue_index: "{{item}}"
+    class_enable: "{{(1).__lshift__(item)}}"
+    expected_matches_target: "{% if total_test_ports_num is defined %}{{(total_test_ports_num | int) * (pfc_queue_indices | length)}}{% endif %}"
+  with_items: "{{pfc_queue_indices}}"