diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml index f50a9961ccf..662ef573f63 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml @@ -1,5 +1,6 @@ #------------------------------------ # Test the PFC restore action +# Iterate the test over a list of lossless queues # 1. Verify restore is detected via syslog entry analysis on the dut # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test #------------------------------------ @@ -49,7 +50,7 @@ - name: Prepare variables required for PFC test set_fact: - pfc_queue_index: 4 + pfc_queue_indices: [4] pfc_frames_number: 100000000 pfc_wd_test_pkt_count: 100 pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}" @@ -57,8 +58,10 @@ peer_mgmt: "{{device_info['mgmtip']}}" testname: functional_test -- set_fact: - class_enable: "{{(1).__lshift__(pfc_queue_index)}}" +- name: Add queue index 3 to pfc_queue_indices when seed is an odd number + set_fact: + pfc_queue_indices: "{{pfc_queue_indices + [3]}}" + when: seed | int is odd - set_fact: peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}" @@ -67,76 +70,9 @@ include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml -- block: - # 1. Verify restore is detected via syslog entry analysis on the dut - - set_fact: - test_expect_file: "expect_pfc_wd_restore" - - - name: Initialize loganalyzer - include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml - - - name: Stop PFC storm on fanout switch - action: apswitch template="{{pfc_wd_storm_stop_template}}" - args: - host: "{{peer_mgmt}}" - login: "{{peer_login}}" - connection: switch - - - name: Wait for queue to recover from PFC storm - pause: - seconds: 1 - - - name: Check if logs contain message that PFC WD restored from deadlock - include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - - - name: Check if logs contain message that PFC WD restored from deadlock - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml - - # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test - - name: "Send packets via {{pfc_wd_test_port}}" - include: roles/test/tasks/ptf_runner.yml - vars: - ptf_test_name: PFC WD test - ptf_test_dir: ptftests - ptf_test_path: pfc_wd.PfcWdTest - ptf_platform: remote - ptf_platform_dir: ptftests - ptf_test_params: - - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_ethernet0_mac_addr}}' - - queue_index='{{pfc_queue_index}}' - - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id[0]}}' - - port_dst='{{pfc_wd_test_port_ids}}' - - ip_dst='{{pfc_wd_test_neighbor_addr}}' - - port_type='{{port_type}}' - - wd_action='forward' - ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - - name: "Send packets to {{pfc_wd_test_port}}" - include: roles/test/tasks/ptf_runner.yml - vars: - ptf_test_name: PFC WD test - ptf_test_dir: ptftests - ptf_test_path: pfc_wd.PfcWdTest - ptf_platform: remote - ptf_platform_dir: ptftests - ptf_test_params: - - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_ethernet0_mac_addr}}' - - queue_index='{{pfc_queue_index}}' - - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_test_port_id}}' - - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - - ip_dst='{{pfc_wd_rx_neighbor_addr}}' - - port_type='{{port_type}}' - - wd_action='forward' - ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - rescue: - - name: Stop PFC storm on fanout switch - action: apswitch template="{{pfc_wd_storm_stop_template}}" - args: - host: "{{peer_mgmt}}" - login: "{{peer_login}}" - connection: switch +- name: Test PFC restore function per queue + include: roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml + vars: + pfc_queue_index: "{{item}}" + class_enable: "{{(1).__lshift__(item)}}" + with_items: "{{pfc_queue_indices}}" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml new file mode 100644 index 00000000000..e7b1e751b0d --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_restore_perq.yml @@ -0,0 +1,81 @@ +#------------------------------------ +# Test the PFC restore action +# 1. Verify restore is detected via syslog entry analysis on the dut +# 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test +#------------------------------------ + +- block: + # 1. Verify restore is detected via syslog entry analysis on the dut + - set_fact: + test_expect_file: "expect_pfc_wd_restore" + + - name: Initialize loganalyzer + include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml + vars: + testname_unique_gen: true + + - name: Stop PFC storm on fanout switch + action: apswitch template="{{pfc_wd_storm_stop_template}}" + args: + host: "{{peer_mgmt}}" + login: "{{peer_login}}" + connection: switch + + - name: Allow enough time for the PFC storm restoration to flow into the syslog + pause: + seconds: 1 + + - name: Check if logs contain message that PFC WD restored from deadlock + include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml + + - name: Check if logs contain message that PFC WD restored from deadlock + include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml + + # 2. Verity the forward action on both ingress and egress via ptf dataplane traffic test + - name: "Send packets via {{pfc_wd_test_port}}" + include: roles/test/tasks/ptf_runner.yml + vars: + ptf_test_name: PFC WD test + ptf_test_dir: ptftests + ptf_test_path: pfc_wd.PfcWdTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - testbed_type='{{testbed_type}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' + - queue_index='{{pfc_queue_index}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='{{pfc_wd_test_port_ids}}' + - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' + - wd_action='forward' + ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + + - name: "Send packets to {{pfc_wd_test_port}}" + include: roles/test/tasks/ptf_runner.yml + vars: + ptf_test_name: PFC WD test + ptf_test_dir: ptftests + ptf_test_path: pfc_wd.PfcWdTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - testbed_type='{{testbed_type}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' + - queue_index='{{pfc_queue_index}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_test_port_id}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' + - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' + - wd_action='forward' + ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + + rescue: + - name: Stop PFC storm on fanout switch + action: apswitch template="{{pfc_wd_storm_stop_template}}" + args: + host: "{{peer_mgmt}}" + login: "{{peer_login}}" + connection: switch diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml index bf0e9015240..10ef88d7c93 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml @@ -1,5 +1,6 @@ #------------------------------------ -# Test the PFC storm action +# Test the PFC storm action on a port +# Iterate the test over a list of lossless queues # 1. Verify storm is detected via syslog entry analysis on the dut # 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test # @@ -51,7 +52,7 @@ - name: Prepare variables required for PFC test set_fact: - pfc_queue_index: 4 + pfc_queue_indices: [4] pfc_frames_number: 100000000 pfc_wd_test_pkt_count: 100 pfc_fanout_interface: "{{neighbors[pfc_wd_test_port]['peerport']}}" @@ -59,8 +60,10 @@ peer_mgmt: "{{device_info['mgmtip']}}" testname: functional_test -- set_fact: - class_enable: "{{(1).__lshift__(pfc_queue_index)}}" +- name: Add queue index 3 to pfc_queue_indices when seed is an odd number + set_fact: + pfc_queue_indices: "{{pfc_queue_indices + [3]}}" + when: seed | int is odd - set_fact: peer_login: "{{switch_login[hwsku_map[peer_hwsku]]}}" @@ -69,90 +72,10 @@ include: roles/test/tasks/pfc_wd/functional_test/set_pfc_storm_templates.yml -- block: - - set_fact: - pfc_gen_file: pfc_gen.py - - - name: Deploy pfc packet generater file to fanout switch - include: roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml - - - name: copy the test to ptf container - copy: src=roles/test/files/ptftests dest=/root - delegate_to: "{{ptf_host}}" - - # 1. Verify storm is detected via syslog entry analysis on the dut - - set_fact: - test_expect_file: "expect_pfc_wd_detect" - test_ignore_file: "ignore_pfc_wd_messages" - - - name: Initialize loganalyzer - include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml - - - name: Generate PFC storm on fanout switch - action: apswitch template="{{pfc_wd_storm_template}}" - args: - host: "{{peer_mgmt}}" - login: "{{peer_login}}" - connection: switch - register: pid - - - debug: msg="{{pid}}" - - - name: Let PFC storm happen for a while - pause: - seconds: 5 - - - name: Check if logs contain message that PFC WD detected storm - include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - - - name: Check if logs contain message that PFC WD detected storm - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml - - # 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test - - name: "check egress drop, tx port {{pfc_wd_test_port}}" - include: roles/test/tasks/ptf_runner.yml - vars: - ptf_test_name: PFC WD test - ptf_test_dir: ptftests - ptf_test_path: pfc_wd.PfcWdTest - ptf_platform: remote - ptf_platform_dir: ptftests - ptf_test_params: - - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_ethernet0_mac_addr}}' - - queue_index='{{pfc_queue_index}}' - - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_rx_port_id[0]}}' - - port_dst='[{{pfc_wd_test_port_id}}]' - - ip_dst='{{pfc_wd_test_neighbor_addr}}' - - port_type='{{port_type}}' - - wd_action='drop' - ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - - name: "Check ingress drop, rx port {{pfc_wd_test_port}}" - include: roles/test/tasks/ptf_runner.yml - vars: - ptf_test_name: PFC WD test - ptf_test_dir: ptftests - ptf_test_path: pfc_wd.PfcWdTest - ptf_platform: remote - ptf_platform_dir: ptftests - ptf_test_params: - - testbed_type='{{testbed_type}}' - - router_mac='{{ansible_ethernet0_mac_addr}}' - - queue_index='{{pfc_queue_index}}' - - pkt_count='{{pfc_wd_test_pkt_count}}' - - port_src='{{pfc_wd_test_port_id}}' - - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' - - ip_dst='{{pfc_wd_rx_neighbor_addr}}' - - port_type='{{port_type}}' - - wd_action='drop' - ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " - - rescue: - - name: Stop PFC storm on fanout switch - action: apswitch template="{{pfc_wd_storm_stop_template}}" - args: - host: "{{peer_mgmt}}" - login: "{{peer_login}}" - connection: switch +- name: Test PFC storm function per queue + include: roles/test/tasks/pfc_wd/functional_test/functional_test_storm_perq.yml + vars: + pfc_queue_index: "{{item}}" + class_enable: "{{(1).__lshift__(item)}}" + expected_matches_target: "{% if total_test_ports_num is defined %}{{(total_test_ports_num | int) * (pfc_queue_indices | length)}}{% endif %}" + with_items: "{{pfc_queue_indices}}" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm_perq.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm_perq.yml new file mode 100644 index 00000000000..5453542773a --- /dev/null +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_storm_perq.yml @@ -0,0 +1,99 @@ +#------------------------------------ +# Test the PFC storm action +# 1. Verify storm is detected via syslog entry analysis on the dut +# 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test +# +# Storm will not be stopped on exiting the yml if test success +#------------------------------------ + +- block: + - set_fact: + pfc_gen_file: pfc_gen.py + + - name: Deploy pfc packet generater file to fanout switch + include: roles/test/tasks/pfc_wd/functional_test/deploy_pfc_pktgen.yml + + - name: copy the test to ptf container + copy: src=roles/test/files/ptftests dest=/root + delegate_to: "{{ptf_host}}" + + # 1. Verify storm is detected via syslog entry analysis on the dut + - set_fact: + test_expect_file: "expect_pfc_wd_detect" + test_ignore_file: "ignore_pfc_wd_messages" + + - name: Initialize loganalyzer + include: roles/test/files/tools/loganalyzer/loganalyzer_init.yml + vars: + testname_unique_gen: true + + - name: Allow enough time for the start marker to flow into the syslog + pause: + seconds: 5 + + - name: Generate PFC storm on fanout switch + action: apswitch template="{{pfc_wd_storm_template}}" + args: + host: "{{peer_mgmt}}" + login: "{{peer_login}}" + connection: switch + when: start_storm is not defined or start_storm == true + + - name: Allow enough time for the PFC storm detection to flow into the syslog + pause: + seconds: 5 + + - name: Check if logs contain message that PFC WD detected storm + include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml + + - name: Check if logs contain message that PFC WD detected storm + include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml + + # 2. Verify the drop action in place on both ingress and egress via ptf dataplane traffic test + - name: "check egress drop, tx port {{pfc_wd_test_port}}" + include: roles/test/tasks/ptf_runner.yml + vars: + ptf_test_name: PFC WD test + ptf_test_dir: ptftests + ptf_test_path: pfc_wd.PfcWdTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - testbed_type='{{testbed_type}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' + - queue_index='{{pfc_queue_index}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_rx_port_id[0]}}' + - port_dst='[{{pfc_wd_test_port_id}}]' + - ip_dst='{{pfc_wd_test_neighbor_addr}}' + - port_type='{{port_type}}' + - wd_action='drop' + ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + + - name: "Check ingress drop, rx port {{pfc_wd_test_port}}" + include: roles/test/tasks/ptf_runner.yml + vars: + ptf_test_name: PFC WD test + ptf_test_dir: ptftests + ptf_test_path: pfc_wd.PfcWdTest + ptf_platform: remote + ptf_platform_dir: ptftests + ptf_test_params: + - testbed_type='{{testbed_type}}' + - router_mac='{{ansible_ethernet0_mac_addr}}' + - queue_index='{{pfc_queue_index}}' + - pkt_count='{{pfc_wd_test_pkt_count}}' + - port_src='{{pfc_wd_test_port_id}}' + - port_dst='[{{pfc_wd_rx_port_id | join(' ')}}]' + - ip_dst='{{pfc_wd_rx_neighbor_addr}}' + - port_type='{{port_type}}' + - wd_action='drop' + ptf_extra_options: "--relax --debug info --log-file /tmp/pfc_wd.PfcWdTest.{{lookup('pipe','date +%Y-%m-%d-%H:%M:%S')}}.log " + + rescue: + - name: Stop PFC storm on fanout switch + action: apswitch template="{{pfc_wd_storm_stop_template}}" + args: + host: "{{peer_mgmt}}" + login: "{{peer_login}}" + connection: switch diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml index 644a9add423..fd11463411b 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/functional_test_warm_reboot.yml @@ -23,3 +23,30 @@ - name: Test PFC WD restore function include: roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml with_dict: "{{select_test_ports}}" + + +# Test case 2: +# PFC storm started and detected before warm-reboot +# On-going storm on warm-reboot emission, and lasts past the warm-reboot finish +# PFC storm stopped and restored after warm-reboot +- block: + - name: Test PFC WD storm function + include: roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml + with_dict: "{{select_test_ports}}" + + - name: Issue warm reboot on the device + include: roles/test/tasks/common_tasks/reboot_sonic.yml + vars: + reboot_type: warm-reboot + + - name: Test PFC WD storm function + include: roles/test/tasks/pfc_wd/functional_test/functional_test_storm.yml + vars: + start_storm: false + start_marker: "NOTICE swss#orchagent: :- setWarmStartState: orchagent warm start state changed to initialized" + total_test_ports_num: "{{select_test_ports | length}}" + with_dict: "{{select_test_ports}}" + + - name: Test PFC WD restore function + include: roles/test/tasks/pfc_wd/functional_test/functional_test_restore.yml + with_dict: "{{select_test_ports}}" diff --git a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages index 391d16ed990..7cff6813972 100644 --- a/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages +++ b/ansible/roles/test/tasks/pfc_wd/functional_test/ignore_pfc_wd_messages @@ -7,3 +7,5 @@ r, ".* snmp.*" r, ".* Trying to remove nonexisting queue from flex counter .*" r, ".* SAI_STATUS_BUFFER_OVERFLOW" r, ".* ERR ntpd.*routing socket reports: No buffer space available.*" +r, ".* ERR syncd.*" +r, ".* syncd .* ERROR +HOST_INTERFACE"