From 3a9995b6213d893f60898183160bbdf22efca153 Mon Sep 17 00:00:00 2001 From: jingwenxie Date: Thu, 11 May 2023 14:03:51 +0800 Subject: [PATCH] [config]Support multi-asic Golden Config override with fix (#2825) ADO: 17746282 #### What I did Support multi-asic Golden Config Override with fix based on https://github.com/sonic-net/sonic-utilities/pull/2738 #### How I did it Add ConfigMgmt support for ASIC validation. Modify override config cli to support multi-asic. #### How to verify it Unit test: ``` tests/config_override_test.py::TestConfigOverrideMultiasic::test_macsec_override PASSED [ 8%] tests/config_override_test.py::TestConfigOverrideMultiasic::test_device_metadata_table_rm PASSED [ 8%] ``` --- config/config_mgmt.py | 19 +++-- config/main.py | 63 ++++++++------- .../multi_asic_dm_rm.json | 11 +++ .../multi_asic_macsec_ov.json | 23 ++++++ tests/config_override_test.py | 77 ++++++++++++++++++- 5 files changed, 159 insertions(+), 34 deletions(-) create mode 100644 tests/config_override_input/multi_asic_dm_rm.json create mode 100644 tests/config_override_input/multi_asic_macsec_ov.json diff --git a/config/config_mgmt.py b/config/config_mgmt.py index a10393c72c..4e3115bd35 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -35,7 +35,8 @@ class ConfigMgmt(): to verify config for the commands which are capable of change in config DB. ''' - def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, sonicYangOptions=0): + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, + sonicYangOptions=0, configdb=None): ''' Initialise the class, --read the config, --load in data tree. @@ -44,6 +45,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, debug (bool): verbose mode. allowTablesWithoutYang (bool): allow tables without yang model in config or not. + configdb: configdb to work on. Returns: void @@ -54,6 +56,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True, self.source = source self.allowTablesWithoutYang = allowTablesWithoutYang self.sonicYangOptions = sonicYangOptions + self.configdb = configdb # logging vars self.SYSLOG_IDENTIFIER = "ConfigMgmt" @@ -194,8 +197,11 @@ def readConfigDB(self): self.sysLog(doPrint=True, msg='Reading data from Redis configDb') # Read from config DB on sonic switch data = dict() - configdb = ConfigDBConnector() - configdb.connect() + if self.configdb is None: + configdb = ConfigDBConnector() + configdb.connect() + else: + configdb = self.configdb sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.db_to_output(configdb.get_config())) self.configdbJsonIn = sonic_cfggen.FormatConverter.to_serialized(data) self.sysLog(syslog.LOG_DEBUG, 'Reading Input from ConfigDB {}'.\ @@ -215,8 +221,11 @@ def writeConfigDB(self, jDiff): ''' self.sysLog(doPrint=True, msg='Writing in Config DB') data = dict() - configdb = ConfigDBConnector() - configdb.connect(False) + if self.configdb is None: + configdb = ConfigDBConnector() + configdb.connect(False) + else: + configdb = self.configdb sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.to_deserialized(jDiff)) self.sysLog(msg="Write in DB: {}".format(data)) configdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data)) diff --git a/config/main.py b/config/main.py index cfe862af52..5b9bd30013 100644 --- a/config/main.py +++ b/config/main.py @@ -1849,36 +1849,45 @@ def override_config_table(db, input_config_db, dry_run): fg='magenta') sys.exit(1) - config_db = db.cfgdb - - # Read config from configDB - current_config = config_db.get_config() - # Serialize to the same format as json input - sonic_cfggen.FormatConverter.to_serialized(current_config) - - updated_config = update_config(current_config, config_input) + cfgdb_clients = db.cfgdb_clients + + for ns, config_db in cfgdb_clients.items(): + # Read config from configDB + current_config = config_db.get_config() + # Serialize to the same format as json input + sonic_cfggen.FormatConverter.to_serialized(current_config) + + if multi_asic.is_multi_asic(): + # Golden Config will use "localhost" to represent host name + if ns == DEFAULT_NAMESPACE: + ns_config_input = config_input["localhost"] + else: + ns_config_input = config_input[ns] + else: + ns_config_input = config_input + updated_config = update_config(current_config, ns_config_input) - yang_enabled = device_info.is_yang_config_validation_enabled(config_db) - if yang_enabled: - # The ConfigMgmt will load YANG and running - # config during initialization. - try: - cm = ConfigMgmt() - cm.validateConfigData() - except Exception as ex: - click.secho("Failed to validate running config. Error: {}".format(ex), fg="magenta") - sys.exit(1) + yang_enabled = device_info.is_yang_config_validation_enabled(config_db) + if yang_enabled: + # The ConfigMgmt will load YANG and running + # config during initialization. + try: + cm = ConfigMgmt(configdb=config_db) + cm.validateConfigData() + except Exception as ex: + click.secho("Failed to validate running config. Error: {}".format(ex), fg="magenta") + sys.exit(1) - # Validate input config - validate_config_by_cm(cm, config_input, "config_input") - # Validate updated whole config - validate_config_by_cm(cm, updated_config, "updated_config") + # Validate input config + validate_config_by_cm(cm, ns_config_input, "config_input") + # Validate updated whole config + validate_config_by_cm(cm, updated_config, "updated_config") - if dry_run: - print(json.dumps(updated_config, sort_keys=True, - indent=4, cls=minigraph_encoder)) - else: - override_config_db(config_db, config_input) + if dry_run: + print(json.dumps(updated_config, sort_keys=True, + indent=4, cls=minigraph_encoder)) + else: + override_config_db(config_db, ns_config_input) def validate_config_by_cm(cm, config_json, jname): diff --git a/tests/config_override_input/multi_asic_dm_rm.json b/tests/config_override_input/multi_asic_dm_rm.json new file mode 100644 index 0000000000..a4c0dd5fa7 --- /dev/null +++ b/tests/config_override_input/multi_asic_dm_rm.json @@ -0,0 +1,11 @@ +{ + "localhost": { + "DEVICE_METADATA": {} + }, + "asic0": { + "DEVICE_METADATA": {} + }, + "asic1": { + "DEVICE_METADATA": {} + } +} diff --git a/tests/config_override_input/multi_asic_macsec_ov.json b/tests/config_override_input/multi_asic_macsec_ov.json new file mode 100644 index 0000000000..ba86f6ef60 --- /dev/null +++ b/tests/config_override_input/multi_asic_macsec_ov.json @@ -0,0 +1,23 @@ +{ + "localhost": { + "MACSEC_PROFILE": { + "profile": { + "key": "value" + } + } + }, + "asic0": { + "MACSEC_PROFILE": { + "profile": { + "key": "value" + } + } + }, + "asic1": { + "MACSEC_PROFILE": { + "profile": { + "key": "value" + } + } + } +} diff --git a/tests/config_override_test.py b/tests/config_override_test.py index 1b058ace13..ca14ae75bb 100644 --- a/tests/config_override_test.py +++ b/tests/config_override_test.py @@ -1,6 +1,7 @@ import os import json import filecmp +import importlib import config.main as config from click.testing import CliRunner @@ -20,6 +21,8 @@ RUNNING_CONFIG_YANG_FAILURE = os.path.join(DATA_DIR, "running_config_yang_failure.json") GOLDEN_INPUT_YANG_FAILURE = os.path.join(DATA_DIR, "golden_input_yang_failure.json") FINAL_CONFIG_YANG_FAILURE = os.path.join(DATA_DIR, "final_config_yang_failure.json") +MULTI_ASIC_MACSEC_OV = os.path.join(DATA_DIR, "multi_asic_macsec_ov.json") +MULTI_ASIC_DEVICE_METADATA_RM = os.path.join(DATA_DIR, "multi_asic_dm_rm.json") # Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') @@ -173,7 +176,7 @@ def test_yang_verification_enabled(self): def is_yang_config_validation_enabled_side_effect(filename): return True - def config_mgmt_side_effect(): + def config_mgmt_side_effect(configdb): return config_mgmt.ConfigMgmt(source=CONFIG_DB_JSON_FILE) db = Db() @@ -232,7 +235,7 @@ def check_yang_verification_failure(self, db, config, running_config, def read_json_file_side_effect(filename): return golden_config - def config_mgmt_side_effect(): + def config_mgmt_side_effect(configdb): return config_mgmt.ConfigMgmt(source=CONFIG_DB_JSON_FILE) # ConfigMgmt will call ConfigDBConnector to load default config_db.json. @@ -257,3 +260,73 @@ def teardown_class(cls): print("TEARDOWN") os.environ["UTILITIES_UNIT_TESTING"] = "0" return + + +class TestConfigOverrideMultiasic(object): + @classmethod + def setup_class(cls): + print("SETUP") + os.environ["UTILITIES_UNIT_TESTING"] = "1" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "multi_asic" + # change to multi asic config + from .mock_tables import dbconnector + from .mock_tables import mock_multi_asic + importlib.reload(mock_multi_asic) + dbconnector.load_namespace_config() + return + + def test_macsec_override(self): + def read_json_file_side_effect(filename): + with open(MULTI_ASIC_MACSEC_OV, "r") as f: + macsec_profile = json.load(f) + return macsec_profile + db = Db() + cfgdb_clients = db.cfgdb_clients + + # The profile_content was copied from MULTI_ASIC_MACSEC_OV, where all + # ns sharing the same content: {"profile": {"key": "value"}} + profile_content = {"profile": {"key": "value"}} + + with mock.patch('config.main.read_json_file', + mock.MagicMock(side_effect=read_json_file_side_effect)): + runner = CliRunner() + result = runner.invoke(config.config.commands["override-config-table"], + ['golden_config_db.json'], obj=db) + assert result.exit_code == 0 + + for ns, config_db in cfgdb_clients.items(): + assert config_db.get_config()['MACSEC_PROFILE'] == profile_content + + def test_device_metadata_table_rm(self): + def read_json_file_side_effect(filename): + with open(MULTI_ASIC_DEVICE_METADATA_RM, "r") as f: + device_metadata = json.load(f) + return device_metadata + db = Db() + cfgdb_clients = db.cfgdb_clients + + for ns, config_db in cfgdb_clients.items(): + assert 'DEVICE_METADATA' in config_db.get_config() + + with mock.patch('config.main.read_json_file', + mock.MagicMock(side_effect=read_json_file_side_effect)): + runner = CliRunner() + result = runner.invoke(config.config.commands["override-config-table"], + ['golden_config_db.json'], obj=db) + assert result.exit_code == 0 + + for ns, config_db in cfgdb_clients.items(): + assert 'DEVICE_METADATA' not in config_db.get_config() + + + @classmethod + def teardown_class(cls): + print("TEARDOWN") + os.environ["UTILITIES_UNIT_TESTING"] = "0" + os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] = "" + # change back to single asic config + from .mock_tables import dbconnector + from .mock_tables import mock_single_asic + importlib.reload(mock_single_asic) + dbconnector.load_namespace_config() + return