From 3f0db4f36205dd8be4b0e32b72a9cef6c4b22a14 Mon Sep 17 00:00:00 2001
From: Liping Xu <108326363+lipxu@users.noreply.github.com>
Date: Thu, 18 Apr 2024 11:26:23 +0800
Subject: [PATCH] [test_fdb_mac_move] Only output error level message into
 syslog if /var/log size limitation  (#12467)

What is the motivation for this PR?
Exception: start-LogAnalyzer-test_fdb_mac_move.2024-02-25-07:03:31 was not found in /var/log
Sometimes, the case failed because the log analyzer cannot locate the log.
The log analyzer records the timestamp at the beginning of the case and analyzes the syslogs to check any error message generated during the case process.
However, some devices with a small disk, have a limitation for the /var/log size.
So during the log rotation, the previous syslog would be deleted, leading to the log analyzer failed to locate the log and return error.

How did you do it?
In this case, the swss logs consume a large space of /var/log. However, the output messages are expected and necessary for debugging in production. and log flooding is typically not a common occurrence in production.
To fix this issue during the case, we need to modify the swss logs's location, but it needs image change.
Therefore the fix introduced in this PR's is a workaround, it check the /var/log available size before test, if no enough space, modify the rsyslog config to only output error messages during the case.

How did you verify/test it?
Run the case.
---
 tests/fdb/test_fdb_mac_move.py | 47 +++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/tests/fdb/test_fdb_mac_move.py b/tests/fdb/test_fdb_mac_move.py
index 21db02e1969..dddb90fded8 100644
--- a/tests/fdb/test_fdb_mac_move.py
+++ b/tests/fdb/test_fdb_mac_move.py
@@ -27,6 +27,50 @@
 ]
 
 
+def modify_rsyslog_severity_level(dut):
+    logger.info('Modify rsyslog severity level to error on dut: {}'.format(dut.hostname))
+    dut.shell("sudo mv /etc/rsyslog.d /etc/rsyslog.d.bak")
+    dut.shell("sudo mkdir /etc/rsyslog.d")
+    dut.shell("sudo echo '*.err    /var/log/syslog' > /etc/rsyslog.d/50_debug.conf")
+    dut.shell("sudo systemctl restart rsyslog")
+    time.sleep(5)
+
+
+def revert_rsyslog_severity_level(dut):
+    logger.info('Revert rsyslog severity level to error on dut: {}'.format(dut.hostname))
+    dut.shell("sudo rm -rf /etc/rsyslog.d")
+    dut.shell("sudo mv /etc/rsyslog.d.bak /etc/rsyslog.d")
+    dut.shell("sudo systemctl restart rsyslog")
+    time.sleep(5)
+
+
+@pytest.fixture(scope="function")
+def fixture_rsyslog_conf_setup_teardown(duthosts, rand_one_dut_hostname):
+    duthost = duthosts[rand_one_dut_hostname]
+
+    # workaround for small disk devices which also has limitation on /var/log size
+    cmd = "df -m"
+    cmd_response_lines = duthost.shell(cmd, module_ignore_errors=True).get('stdout_lines', [])
+    logger.debug("cmd {} rsp {}".format(cmd, cmd_response_lines))
+
+    available_size = 0
+    for line in cmd_response_lines:
+        if "/var/log" in line:
+            available_size = int(line.split()[3])
+            break
+
+    if available_size < 400:
+        modify_rsyslog_severity_level(duthost)
+        rsyslog_severity_level_modified = True
+    else:
+        rsyslog_severity_level_modified = False
+
+    yield
+
+    if rsyslog_severity_level_modified:
+        revert_rsyslog_severity_level(duthost)
+
+
 def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count):
     """
     :param ptfadapter: PTF adapter object
@@ -56,7 +100,8 @@ def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count):
     return fdb
 
 
-def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level):
+def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level,
+                      fixture_rsyslog_conf_setup_teardown):
     # Perform FDB clean up before each test
     fdb_cleanup(duthosts, rand_one_dut_hostname)