Merged PR 691: Block lab devices from sending syslog to production

Block lab devices from sending syslog to production

The internal SONiC image has production syslog server hard coded.
Sometimes, a bad image may flood too many syslog messages and overwhelm
the production syslog server.

This change added code to comment out the production syslog server
from rsyslog configuration file and template file.

Signed-off-by: Xin Wang <[email protected]>

ansible/config_sonic_basedon_testbed.yml

@@ -486,13 +486,27 @@
         become: true
         shell: systemctl start topology.service
         when: start_topo_service is defined and start_topo_service|bool == true
-      
+
       # Remove me when the sai_remap_prio_on_tnl_egress=1 is integrated into image
       - name: Patch bcm.config to workaround BCM SAI issue
         bcm_sai_patch:
         ignore_errors: yes
         become: true
 
+      - name: Patch rsyslog.conf to stop sending syslog to production
+        lineinfile:
+          path: "{{ rsyslog_conf_path }}"
+          state: present
+          backrefs: yes
+          regexp: '(^[^#]*@\[10\.20\.6\.16\]:514)'
+          line: '# \g<1>'
+        loop:
+          - /usr/share/sonic/templates/rsyslog.conf.j2
+          - /etc/rsyslog.conf
+        loop_control:
+          loop_var: rsyslog_conf_path
+        become: true
+
       - name: execute cli "config load_minigraph -y" to apply new minigraph
         become: true
         shell: config load_minigraph -y

ansible/upgrade_sonic.yml

@@ -131,6 +131,24 @@
     - name: Wait for SONiC initialization
       pause: seconds=60
 
+    - name: Patch rsyslog.conf to stop sending syslog to production
+      lineinfile:
+        path: "{{ rsyslog_conf_path }}"
+        state: present
+        backrefs: yes
+        regexp: '(^[^#]*@\[10\.20\.6\.16\]:514)'
+        line: '# \g<1>'
+      loop:
+        - /usr/share/sonic/templates/rsyslog.conf.j2
+        - /etc/rsyslog.conf
+      loop_control:
+        loop_var: rsyslog_conf_path
+      become: true
+
+    - name: Restart rsyslog service
+      shell: systemctl restart rsyslog
+      become: true
+
     - name: Set all bgp interfaces admin-up
       become: true
       shell: config bgp startup all

tests/common/helpers/dut_utils.py

@@ -266,12 +266,12 @@ def verify_orchagent_running_or_assert(duthost):
     """
     Verifies that orchagent is running, asserts otherwise
 
-    Args: 
+    Args:
         duthost: Device Under Test (DUT)
     """
-   
+
     def _orchagent_running():
-        cmds = 'docker exec swss supervisorctl status orchagent' 
+        cmds = 'docker exec swss supervisorctl status orchagent'
         output = duthost.shell(cmds, module_ignore_errors=True)
         pytest_assert(not output['rc'], "Unable to check orchagent status output")
         return 'RUNNING' in output['stdout']
@@ -280,3 +280,20 @@ def _orchagent_running():
         wait_until(120, 10, 0, _orchagent_running),
         "Orchagent is not running"
     )
+
+
+def patch_rsyslog(duthost):
+    """Patch rsyslog configuration to stop sending syslogs to production syslog server
+
+    Args:
+        duthost (obj): Device Under Test (DUT)
+    """
+    for conf in ["/usr/share/sonic/templates/rsyslog.conf.j2", "/etc/rsyslog.conf"]:
+        duthost.lineinfile(
+            path=conf,
+            state="present",
+            backrefs=True,
+            regexp="(^[^#]*@\[10\.20\.6\.16\]:514)",
+            line="# \g<1>"
+        )
+    duthost.shell("systemctl restart rsyslog")

tests/metadata-scripts/test_metadata_upgrade_path.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from tests.ptf_runner import ptf_runner
 from tests.common.helpers.assertions import pytest_assert
+from tests.common.helpers.dut_utils import patch_rsyslog
 from tests.common.platform.ssh_utils import prepare_testbed_ssh_keys
 from tests.common import reboot
 from tests.common.reboot import get_reboot_cause, reboot_ctrl_dict
@@ -90,6 +91,7 @@ def sonic_update_firmware(duthost, image_url, upgrade_type):
     duthost.command("chmod +x /tmp/anpscripts/update_firmware")
     duthost.command("/usr/bin/sudo /tmp/anpscripts/update_firmware {} UPDATE_MLNX_CPLD_FW={}".format(
         image_name, UPDATE_MLNX_CPLD_FW))
+    patch_rsyslog(duthost)
 
     return out['stdout'].rstrip('\n')
 
@@ -138,6 +140,8 @@ def run_upgrade_test(duthost, localhost, ptfhost, from_image, to_image,
             advanceboot_loganalyzer=advanceboot_loganalyzer, allow_fail=allow_fail)
         advancedReboot.runRebootTestcase(prebootList=sad_preboot_list, inbootList=sad_inboot_list)
 
+    patch_rsyslog(duthost)
+
     if create_hole:
         ptfhost.shell('supervisorctl stop ferret')
 

tests/test_posttest.py

@@ -55,34 +55,3 @@ def test_recover_rsyslog_rate_limit(duthosts, enum_dut_hostname):
         if 'enabled' not in state:
             continue
         duthost.modify_syslog_rate_limit(feature_name, rl_option='enable')
-
-def test_unblock_rsyslog_server(duthosts, enum_dut_hostname):
-    """
-    Unblock rsyslog server 10.20.6.16:514
-    """
-
-    """
-    FIXME: we shouldn't restore configuration until spamer is stopped
-
-    # Recover to backup file
-    RSYS_CONF = "/etc/rsyslog.conf"
-    RSYS_CONF_BAK = RSYS_CONF + ".bak"
-    RSYS_TEMPL = "/usr/share/sonic/templates/rsyslog.conf.j2"
-    RSYS_TEMPL_BAK = RSYS_TEMPL + ".bak"
-
-    duthost = duthosts[enum_dut_hostname]
-
-    if duthost.stat(path=RSYS_TEMPL_BAK)['stat']['exists']:
-        cmds = [
-            "mv {} {}".format(RSYS_TEMPL_BAK, RSYS_TEMPL),
-        ]
-        duthost.shell_cmds(cmds=cmds)
-
-    if duthost.stat(path=RSYS_CONF_BAK)['stat']['exists']:
-        cmds = [
-            "mv {} {}".format(RSYS_CONF_BAK, RSYS_CONF),
-            "systemctl restart rsyslog"
-        ]
-        duthost.shell_cmds(cmds=cmds)
-    """
-    pass

tests/test_pretest.py

@@ -361,23 +361,3 @@ def test_update_buffer_template(duthosts, enum_dut_hostname, localhost):
     if buf_temp_changed:
         logging.info("Executing load minigraph ...")
         config_reload(duthost, config_source='minigraph')
-
-def test_block_rsyslog_server(duthosts, enum_dut_hostname):
-    """
-    Block rsyslog server 10.20.6.16:514
-    """
-    # Make a backup file
-    RSYS_CONF = "/etc/rsyslog.conf"
-    RSYS_TEMPL = "/usr/share/sonic/templates/rsyslog.conf.j2"
-    PATTERN = r"*.* @10.20.6.16:514"
-
-    cmds = [
-        "cp {} {}".format(RSYS_CONF, RSYS_CONF + ".bak"),
-        "cp {} {}".format(RSYS_TEMPL, RSYS_TEMPL + ".bak"),
-        r"sed -i 's/" + PATTERN + r"/#" + PATTERN + r"/g' " + RSYS_CONF,
-        r"sed -i 's/" + PATTERN + r"/#" + PATTERN + r"/g' " + RSYS_TEMPL,
-        "systemctl restart rsyslog"
-    ]
-
-    duthost = duthosts[enum_dut_hostname]
-    duthost.shell_cmds(cmds=cmds)

tests/upgrade_path/test_upgrade_path.py

@@ -5,6 +5,7 @@
 import random
 import logging
 from tests.common.helpers.assertions import pytest_assert
+from tests.common.helpers.dut_utils import patch_rsyslog
 from tests.common import reboot
 from tests.common.reboot import get_reboot_cause
 from tests.common.reboot import REBOOT_TYPE_COLD
@@ -63,6 +64,7 @@ def test_upgrade_path(localhost, duthosts, ptfhost, rand_one_dut_hostname, nbrho
             # Perform a cold reboot
             logger.info("Cold reboot the DUT to make the base image as current")
             reboot(duthost, localhost)
+            patch_rsyslog(duthost)
             check_sonic_version(duthost, target_version)
 
             # Install target image
@@ -75,6 +77,7 @@ def test_upgrade_path(localhost, duthosts, ptfhost, rand_one_dut_hostname, nbrho
                 advancedReboot = get_advanced_reboot(rebootType=get_reboot_command(duthost, upgrade_type),\
                     advanceboot_loganalyzer=advanceboot_loganalyzer)
                 advancedReboot.runRebootTestcase()
+            patch_rsyslog(duthost)
             reboot_cause = get_reboot_cause(duthost)
             logger.info("Check reboot cause. Expected cause {}".format(upgrade_type))
             pytest_assert(reboot_cause == upgrade_type, "Reboot cause {} did not match the trigger - {}".format(reboot_cause, upgrade_type))
@@ -100,6 +103,7 @@ def test_warm_upgrade_sad_path(localhost, duthosts, ptfhost, rand_one_dut_hostna
             # Perform a cold reboot
             logger.info("Cold reboot the DUT to make the base image as current")
             reboot(duthost, localhost)
+            patch_rsyslog(duthost)
             check_sonic_version(duthost, target_version)
 
             # Install target image
@@ -112,6 +116,7 @@ def test_warm_upgrade_sad_path(localhost, duthosts, ptfhost, rand_one_dut_hostna
                 prebootList=sad_preboot_list,
                 inbootList=sad_inboot_list
             )
+            patch_rsyslog(duthost)
             reboot_cause = get_reboot_cause(duthost)
             logger.info("Check reboot cause. Expected cause {}".format(upgrade_type))
             pytest_assert(reboot_cause == upgrade_type, "Reboot cause {} did not match the trigger - {}".format(reboot_cause, upgrade_type))