[TACACS] Improve TACACS run command on IPV6 failed issue. (#12819)

Improve TACACS run command on IPV6 failed issue.

#### Why I did it
Tacplus server crash when receive authorization request from IPV6 address.

#### How I did it
Check TACACS server status after run command with IPV6 address.

#### How to verify it
Pass all test case.

#### Description for the changelog
Improve TACACS run command on IPV6 failed issue.

tests/tacacs/test_ro_user.py

@@ -1,7 +1,7 @@
 import pytest
 import time
 from tests.common.helpers.assertions import pytest_assert
-from .utils import check_output
+from .utils import check_output, tacacs_running, start_tacacs_server
 
 import logging
 
@@ -77,6 +77,21 @@ def wait_for_tacacs(localhost, remote_ip, username, password):
                 current_attempt += 1
 
 
+def ssh_remote_run_retry(localhost, dutip, ptfhost, user, password, command, retry_count=3):
+    while retry_count > 0:
+        res = ssh_remote_run(localhost, dutip, user,
+                             password, command)
+
+        # TACACS server randomly crash after receive authorization request from IPV6
+        if not tacacs_running(ptfhost):
+            start_tacacs_server(ptfhost)
+            retry_count -= 1
+        else:
+            return res
+
+    pytest_assert(False, "cat command failed because TACACS server not running")
+
+
 def test_ro_user(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs):
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
     dutip = duthost.mgmt_ip
@@ -86,13 +101,16 @@ def test_ro_user(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_c
     check_output(res, 'test', 'remote_user')
 
 
-def test_ro_user_ipv6(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs_v6):
+def test_ro_user_ipv6(localhost, ptfhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs_v6):
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
     dutip = duthost.mgmt_ip
-    res = ssh_remote_run(localhost, dutip, tacacs_creds['tacacs_ro_user'],
-                         tacacs_creds['tacacs_ro_user_passwd'], 'cat /etc/passwd')
 
-    check_output(res, 'test', 'remote_user')
+    res = ssh_remote_run_retry(localhost, dutip, ptfhost,
+                               tacacs_creds['tacacs_ro_user'],
+                               tacacs_creds['tacacs_ro_user_passwd'],
+                               "cat /etc/passwd")
+
+    check_output(res, 'testadmin', 'remote_user_su')
 
 
 def test_ro_user_allowed_command(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs):
@@ -173,7 +191,8 @@ def test_ro_user_allowed_command(localhost, duthosts, enum_rand_one_per_hwsku_ho
                                           " 'sudo sonic-installer list' is banned")
 
 
-def test_ro_user_banned_by_sudoers_command(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs):
+def test_ro_user_banned_by_sudoers_command(localhost, duthosts, enum_rand_one_per_hwsku_hostname,
+                                           tacacs_creds, check_tacacs):
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
     dutip = duthost.mgmt_ip
 

tests/tacacs/test_rw_user.py

@@ -1,6 +1,6 @@
 import pytest
 
-from .test_ro_user import ssh_remote_run
+from .test_ro_user import ssh_remote_run, ssh_remote_run_retry
 from .utils import check_output
 
 pytestmark = [
@@ -21,12 +21,15 @@ def test_rw_user(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_c
     check_output(res, 'testadmin', 'remote_user_su')
 
 
-def test_rw_user_ipv6(localhost, duthosts, enum_rand_one_per_hwsku_hostname, tacacs_creds, check_tacacs_v6):
+def test_rw_user_ipv6(localhost, duthosts, ptfhost, enum_rand_one_per_hwsku_hostname,
+                      tacacs_creds, check_tacacs_v6):
     """test tacacs rw user
     """
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
     dutip = duthost.mgmt_ip
-    res = ssh_remote_run(localhost, dutip, tacacs_creds['tacacs_rw_user'],
-                         tacacs_creds['tacacs_rw_user_passwd'], "cat /etc/passwd")
+    res = ssh_remote_run_retry(localhost, dutip, ptfhost,
+                               tacacs_creds['tacacs_rw_user'],
+                               tacacs_creds['tacacs_rw_user_passwd'],
+                               "cat /etc/passwd")
 
     check_output(res, 'testadmin', 'remote_user_su')

tests/tacacs/utils.py

@@ -34,11 +34,12 @@ def check_all_services_status(ptfhost):
     logger.info(res["stdout_lines"])
 
 
-def start_tacacs_server(ptfhost):
-    def tacacs_running(ptfhost):
-        out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"]
-        return "tacacs+ running" in out
+def tacacs_running(ptfhost):
+    out = ptfhost.command("service tacacs_plus status", module_ignore_errors=True)["stdout"]
+    return "tacacs+ running" in out
 
+
+def start_tacacs_server(ptfhost):
     ptfhost.command("service tacacs_plus restart", module_ignore_errors=True)
     return wait_until(5, 1, 0, tacacs_running, ptfhost)