From c116cafa8ba2c2da2a5cdb006176ceeee3288ee0 Mon Sep 17 00:00:00 2001 From: liuh-80 <liuh@microsoft.com> Date: Wed, 12 Jun 2024 02:50:54 +0000 Subject: [PATCH 1/4] Improve load_mingraph to wait eth0 restart before exist --- config/main.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/config/main.py b/config/main.py index 30b14e025e..67c5860abe 100644 --- a/config/main.py +++ b/config/main.py @@ -898,10 +898,36 @@ def _reset_failed_services(): for service in _get_sonic_services(): clicommon.run_command(['systemctl', 'reset-failed', str(service)]) +def get_service_finish_timestamp(service): + out, _ = clicommon.run_command(['sudo', 'systemctl', 'show', '--no-pager', service, '-p', 'ExecMainExitTimestamp', '--value'], return_cmd=True) + return out.strip(' \t\n\r') + +def wait_service_restart_finish(service, last_timestamp, timeout = 30): + start_time = time.time() + elapsed_time = 0 + while elapsed_time < timeout: + current_timestamp = get_service_finish_timestamp(service) + if current_timestamp and (current_timestamp != last_timestamp): + return + + time.sleep(1) + elapsed_time = time.time() - start_time + + log.log_warning("Service: {} does not restart in {} seconds, stop waiting".format(service, timeout)) + def _restart_services(): + last_interface_config_timestamp = get_service_finish_timestamp('interfaces-config') + last_networking_timestamp = get_service_finish_timestamp('networking') + click.echo("Restarting SONiC target ...") clicommon.run_command(['sudo', 'systemctl', 'restart', 'sonic.target']) + # These service will restart eth0 and cause device lost network for 10 seconds + # When enable TACACS, every remote user commands will authorize by TACACS service via network + # If load_minigraph exit before eth0 restart, commands after load_minigraph may failed + wait_service_restart_finish('interfaces-config', last_interface_config_timestamp) + wait_service_restart_finish('networking', last_networking_timestamp) + try: subprocess.check_call(['sudo', 'monit', 'status'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) click.echo("Enabling container monitoring ...") From 004e92decfcea65284c7c8ca090d4e2c38b50577 Mon Sep 17 00:00:00 2001 From: liuh-80 <liuh@microsoft.com> Date: Wed, 12 Jun 2024 03:11:08 +0000 Subject: [PATCH 2/4] Fix code format --- config/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/config/main.py b/config/main.py index 67c5860abe..218b00462d 100644 --- a/config/main.py +++ b/config/main.py @@ -898,10 +898,20 @@ def _reset_failed_services(): for service in _get_sonic_services(): clicommon.run_command(['systemctl', 'reset-failed', str(service)]) + def get_service_finish_timestamp(service): - out, _ = clicommon.run_command(['sudo', 'systemctl', 'show', '--no-pager', service, '-p', 'ExecMainExitTimestamp', '--value'], return_cmd=True) + out, _ = clicommon.run_command(['sudo', + 'systemctl', + 'show', + '--no-pager', + service, + '-p', + 'ExecMainExitTimestamp', + '--value'], + return_cmd=True) return out.strip(' \t\n\r') + def wait_service_restart_finish(service, last_timestamp, timeout = 30): start_time = time.time() elapsed_time = 0 @@ -915,6 +925,7 @@ def wait_service_restart_finish(service, last_timestamp, timeout = 30): log.log_warning("Service: {} does not restart in {} seconds, stop waiting".format(service, timeout)) + def _restart_services(): last_interface_config_timestamp = get_service_finish_timestamp('interfaces-config') last_networking_timestamp = get_service_finish_timestamp('networking') From fe5994fe4878d50e631e3d71adc55bce8d384643 Mon Sep 17 00:00:00 2001 From: liuh-80 <liuh@microsoft.com> Date: Wed, 12 Jun 2024 04:14:00 +0000 Subject: [PATCH 3/4] Fix test case --- config/main.py | 2 +- tests/config_test.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config/main.py b/config/main.py index 218b00462d..bf03962d71 100644 --- a/config/main.py +++ b/config/main.py @@ -912,7 +912,7 @@ def get_service_finish_timestamp(service): return out.strip(' \t\n\r') -def wait_service_restart_finish(service, last_timestamp, timeout = 30): +def wait_service_restart_finish(service, last_timestamp, timeout=30): start_time = time.time() elapsed_time = 0 while elapsed_time < timeout: diff --git a/tests/config_test.py b/tests/config_test.py index f69f799561..5ccfaf5e9e 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -7,6 +7,7 @@ import json import jsonpatch import sys +import time import unittest import ipaddress import shutil @@ -244,6 +245,10 @@ def mock_run_command_side_effect(*args, **kwargs): return 'enabled', 0 elif command == 'cat /var/run/dhclient.eth0.pid': return '101', 0 + elif command == 'sudo systemctl show --no-pager interfaces-config -p ExecMainExitTimestamp --value': + return f'{time.localtime()}', 0 + elif command == 'sudo systemctl show --no-pager networking -p ExecMainExitTimestamp --value': + return f'{time.localtime()}', 0 else: return '', 0 @@ -656,7 +661,7 @@ def test_load_minigraph(self, get_cmd_module, setup_single_broadcom_asic): assert "\n".join([l.rstrip() for l in result.output.split('\n')]) == load_minigraph_command_output # Verify "systemctl reset-failed" is called for services under sonic.target mock_run_command.assert_any_call(['systemctl', 'reset-failed', 'swss']) - assert mock_run_command.call_count == 8 + assert mock_run_command.call_count == 12 @mock.patch('sonic_py_common.device_info.get_paths_to_platform_and_hwsku_dirs', mock.MagicMock(return_value=(load_minigraph_platform_path, None))) def test_load_minigraph_platform_plugin(self, get_cmd_module, setup_single_broadcom_asic): @@ -671,7 +676,7 @@ def test_load_minigraph_platform_plugin(self, get_cmd_module, setup_single_broad assert "\n".join([l.rstrip() for l in result.output.split('\n')]) == load_minigraph_platform_plugin_command_output # Verify "systemctl reset-failed" is called for services under sonic.target mock_run_command.assert_any_call(['systemctl', 'reset-failed', 'swss']) - assert mock_run_command.call_count == 8 + assert mock_run_command.call_count == 12 @mock.patch('sonic_py_common.device_info.get_paths_to_platform_and_hwsku_dirs', mock.MagicMock(return_value=(load_minigraph_platform_false_path, None))) def test_load_minigraph_platform_plugin_fail(self, get_cmd_module, setup_single_broadcom_asic): From 18ee290d09872269c186f99a06f29e9375eed3d5 Mon Sep 17 00:00:00 2001 From: liuh-80 <liuh@microsoft.com> Date: Wed, 12 Jun 2024 05:10:48 +0000 Subject: [PATCH 4/4] Fix test case --- tests/config_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/config_test.py b/tests/config_test.py index 5ccfaf5e9e..9d0b277d81 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,4 +1,5 @@ import copy +import datetime import pytest import filecmp import importlib @@ -7,7 +8,6 @@ import json import jsonpatch import sys -import time import unittest import ipaddress import shutil @@ -246,9 +246,9 @@ def mock_run_command_side_effect(*args, **kwargs): elif command == 'cat /var/run/dhclient.eth0.pid': return '101', 0 elif command == 'sudo systemctl show --no-pager interfaces-config -p ExecMainExitTimestamp --value': - return f'{time.localtime()}', 0 + return f'{datetime.datetime.now()}', 0 elif command == 'sudo systemctl show --no-pager networking -p ExecMainExitTimestamp --value': - return f'{time.localtime()}', 0 + return f'{datetime.datetime.now()}', 0 else: return '', 0