From ec9732aa3ba706663c270b1a49bcce2b79b54403 Mon Sep 17 00:00:00 2001 From: SuvarnaMeenakshi <50386592+SuvarnaMeenakshi@users.noreply.github.com> Date: Fri, 27 May 2022 16:28:33 -0700 Subject: [PATCH] [202012][multi-asic][sonic-config-engine]: Get PORT table from namespace config db (#10475) Why I did it Cherry-pick of: #7632 portconfig.py gets PORT table from config_db if it is present. If not, port_config.ini files are parsed. For multi-asic platform, if namespace is passed to get_port_config(), config_db connection was done to host namespace always and not to asic specific namespace. Provides fix for: #7161 How I did it Modify db connection function to connect to namespace config_db. How to verify it Unit-test passed. Verified on multi-asic VS platform. --- src/sonic-config-engine/minigraph.py | 9 +- src/sonic-config-engine/portconfig.py | 39 ++- .../tests/mock_tables/__init__.py | 0 .../tests/mock_tables/asic0/config_db.json | 98 ++++++++ .../mock_tables/asic0/database_config.json | 57 +++++ .../tests/mock_tables/asic1/config_db.json | 2 + .../mock_tables/asic1/database_config.json | 57 +++++ .../tests/mock_tables/asic2/config_db.json | 2 + .../mock_tables/asic2/database_config.json | 57 +++++ .../tests/mock_tables/asic3/config_db.json | 2 + .../mock_tables/asic3/database_config.json | 57 +++++ .../tests/mock_tables/config_db.json | 236 ++++++++++++++++++ .../tests/mock_tables/database_config.json | 57 +++++ .../tests/mock_tables/database_global.json | 24 ++ .../tests/mock_tables/dbconnector.py | 149 +++++++++++ src/sonic-config-engine/tests/test_cfggen.py | 48 +++- src/sonic-config-engine/tests/test_j2files.py | 2 + .../tests/test_multinpu_cfggen.py | 17 ++ 18 files changed, 897 insertions(+), 16 deletions(-) create mode 100644 src/sonic-config-engine/tests/mock_tables/__init__.py create mode 100644 src/sonic-config-engine/tests/mock_tables/asic0/config_db.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic0/database_config.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic1/config_db.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic1/database_config.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic2/config_db.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic2/database_config.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic3/config_db.json create mode 100644 src/sonic-config-engine/tests/mock_tables/asic3/database_config.json create mode 100644 src/sonic-config-engine/tests/mock_tables/config_db.json create mode 100644 src/sonic-config-engine/tests/mock_tables/database_config.json create mode 100644 src/sonic-config-engine/tests/mock_tables/database_global.json create mode 100644 src/sonic-config-engine/tests/mock_tables/dbconnector.py diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 60a28ed19b3e..b9a6b5bf7e15 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -12,7 +12,6 @@ from portconfig import get_port_config -from sonic_py_common.multi_asic import get_asic_id_from_name from sonic_py_common.interface import backplane_prefix # TODO: Remove this once we no longer support Python 2 @@ -1160,12 +1159,6 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw local_devices = [] kube_data = {} - # hostname is the asic_name, get the asic_id from the asic_name - if asic_name is not None: - asic_id = get_asic_id_from_name(asic_name) - else: - asic_id = None - hwsku_qn = QName(ns, "HwSku") hostname_qn = QName(ns, "Hostname") docker_routing_config_mode_qn = QName(ns, "DockerRoutingConfigMode") @@ -1177,7 +1170,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw if child.tag == str(docker_routing_config_mode_qn): docker_routing_config_mode = child.text - (ports, alias_map, alias_asic_map) = get_port_config(hwsku=hwsku, platform=platform, port_config_file=port_config_file, asic=asic_id, hwsku_config_file=hwsku_config_file) + (ports, alias_map, alias_asic_map) = get_port_config(hwsku=hwsku, platform=platform, port_config_file=port_config_file, asic_name=asic_name, hwsku_config_file=hwsku_config_file) port_alias_map.update(alias_map) port_alias_asic_map.update(alias_asic_map) diff --git a/src/sonic-config-engine/portconfig.py b/src/sonic-config-engine/portconfig.py index 478198e5af84..b2d28b5150b8 100644 --- a/src/sonic-config-engine/portconfig.py +++ b/src/sonic-config-engine/portconfig.py @@ -5,11 +5,24 @@ import re import sys - from swsscommon.swsscommon import ConfigDBConnector + from swsscommon import swsscommon from sonic_py_common import device_info + from sonic_py_common.multi_asic import get_asic_id_from_name except ImportError as e: raise ImportError("%s - required module not found" % str(e)) +try: + if os.environ["CFGGEN_UNIT_TESTING"] == "2": + modules_path = os.path.join(os.path.dirname(__file__), ".") + tests_path = os.path.join(modules_path, "tests") + sys.path.insert(0, modules_path) + sys.path.insert(0, tests_path) + import mock_tables.dbconnector + mock_tables.dbconnector.load_namespace_config() + +except KeyError: + pass + # Global Variable PLATFORM_ROOT_PATH = '/usr/share/sonic/device' PLATFORM_ROOT_PATH_DOCKER = '/usr/share/sonic/platform' @@ -46,11 +59,16 @@ def readJson(filename): print("error occurred while parsing json: {}".format(sys.exc_info()[1])) return None -def db_connect_configdb(): +def db_connect_configdb(namespace=None): """ Connect to configdb """ - config_db = ConfigDBConnector() + try: + if namespace is not None: + swsscommon.SonicDBConfig.load_sonic_global_db_config(namespace=namespace) + config_db = swsscommon.ConfigDBConnector(use_unix_socket_path=True, namespace=namespace) + except Exception as e: + return None if config_db is None: return None try: @@ -78,8 +96,8 @@ def get_hwsku_file_name(hwsku=None, platform=None): return candidate return None -def get_port_config(hwsku=None, platform=None, port_config_file=None, hwsku_config_file=None, asic=None): - config_db = db_connect_configdb() +def get_port_config(hwsku=None, platform=None, port_config_file=None, hwsku_config_file=None, asic_name=None): + config_db = db_connect_configdb(asic_name) # If available, Read from CONFIG DB first if config_db is not None and port_config_file is None: @@ -89,11 +107,18 @@ def get_port_config(hwsku=None, platform=None, port_config_file=None, hwsku_conf port_alias_map = {} port_alias_asic_map = {} for intf_name in ports.keys(): - port_alias_map[ports[intf_name]["alias"]] = intf_name + if "alias" in ports[intf_name]: + port_alias_map[ports[intf_name]["alias"]] = intf_name return (ports, port_alias_map, port_alias_asic_map) + if asic_name is not None: + asic_id = str(get_asic_id_from_name(asic_name)) + else: + asic_id = None + if not port_config_file: - port_config_file = device_info.get_path_to_port_config_file(hwsku, asic) + port_config_file = device_info.get_path_to_port_config_file(hwsku, asic_id) + if not port_config_file: return ({}, {}, {}) diff --git a/src/sonic-config-engine/tests/mock_tables/__init__.py b/src/sonic-config-engine/tests/mock_tables/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/sonic-config-engine/tests/mock_tables/asic0/config_db.json b/src/sonic-config-engine/tests/mock_tables/asic0/config_db.json new file mode 100644 index 000000000000..ab29ea517ad9 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic0/config_db.json @@ -0,0 +1,98 @@ +{ + "PORT|Ethernet0": { + "index": "0", + "lanes": "33,34,35,36", + "description": "01T2:Ethernet1:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Ethernet1/1", + "admin_status": "up", + "role": "Ext", + "speed": "40000", + "asic_port_name": "Eth0-ASIC0" + }, + "PORT|Ethernet4": { + "index": "1", + "lanes": "29,30,31,32", + "description": "01T2:Ethernet2:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Ethernet1/2", + "admin_status": "up", + "role": "Ext", + "speed": "40000", + "asic_port_name": "Eth1-ASIC0" + }, + "PORT|Ethernet8": { + "index": "2", + "lanes": "41,42,43,44", + "description": "Ethernet1/3:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Ethernet1/3", + "admin_status": "up", + "role": "Ext", + "speed": "40000", + "asic_port_name": "Eth2-ASIC0" + }, + "PORT|Ethernet12": { + "index": "3", + "lanes": "37,38,39,40", + "description": "Ethernet1/4:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Ethernet1/4", + "admin_status": "up", + "role": "Ext", + "speed": "40000", + "asic_port_name": "Eth3-ASIC0" + }, + "PORT|Ethernet-BP0": { + "index": "0", + "lanes": "13,14,15,16", + "description": "ASIC2:Eth0-ASIC2:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Eth4-ASIC0", + "admin_status": "up", + "role": "Int", + "speed": "40000", + "asic_port_name": "Eth4-ASIC0" + }, + "PORT|Ethernet-BP4": { + "index": "1", + "lanes": "17,18,19,20", + "description": "ASIC2:Eth1-ASIC2:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Eth5-ASIC0", + "admin_status": "up", + "role": "Int", + "speed": "40000", + "asic_port_name": "Eth5-ASIC0" + }, + "PORT|Ethernet-BP8": { + "index": "2", + "lanes": "21,22,23,24", + "description": "ASIC3:Eth0-ASIC3:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Eth6-ASIC0", + "admin_status": "up", + "role": "Int", + "speed": "40000", + "asic_port_name": "Eth6-ASIC0" + }, + "PORT|Ethernet-BP12": { + "index": "3", + "lanes": "25,26,27,28", + "description": "ASIC3:Eth1-ASIC3:config_db", + "pfc_asym": "off", + "mtu": "9100", + "alias": "Eth7-ASIC0", + "admin_status": "up", + "role": "Int", + "speed": "40000", + "asic_port_name": "Eth7-ASIC0" + } +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic0/database_config.json b/src/sonic-config-engine/tests/mock_tables/asic0/database_config.json new file mode 100644 index 000000000000..33fdeb2d7092 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic0/database_config.json @@ -0,0 +1,57 @@ +{ + "INSTANCES": { + "redis": { + "hostname" : "127.0.0.1", + "port" : 6379, + "unix_socket_path" : "/var/run/redis/redis.sock" + } + }, + "DATABASES" : { + "APPL_DB" : { + "id" : 0, + "separator": ":", + "instance" : "redis" + }, + "ASIC_DB" : { + "id" : 1, + "separator": ":", + "instance" : "redis" + }, + "COUNTERS_DB" : { + "id" : 2, + "separator": ":", + "instance" : "redis" + }, + "LOGLEVEL_DB" : { + "id" : 3, + "separator": ":", + "instance" : "redis" + }, + "CONFIG_DB" : { + "id" : 4, + "separator": "|", + "instance" : "redis" + }, + "PFC_WD_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "FLEX_COUNTER_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "STATE_DB" : { + "id" : 6, + "separator": "|", + "instance" : "redis" + }, + "SNMP_OVERLAY_DB" : { + "id" : 7, + "separator": "|", + "instance" : "redis" + } + }, + "VERSION" : "1.1" +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic1/config_db.json b/src/sonic-config-engine/tests/mock_tables/asic1/config_db.json new file mode 100644 index 000000000000..2c63c0851048 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic1/config_db.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic1/database_config.json b/src/sonic-config-engine/tests/mock_tables/asic1/database_config.json new file mode 100644 index 000000000000..33fdeb2d7092 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic1/database_config.json @@ -0,0 +1,57 @@ +{ + "INSTANCES": { + "redis": { + "hostname" : "127.0.0.1", + "port" : 6379, + "unix_socket_path" : "/var/run/redis/redis.sock" + } + }, + "DATABASES" : { + "APPL_DB" : { + "id" : 0, + "separator": ":", + "instance" : "redis" + }, + "ASIC_DB" : { + "id" : 1, + "separator": ":", + "instance" : "redis" + }, + "COUNTERS_DB" : { + "id" : 2, + "separator": ":", + "instance" : "redis" + }, + "LOGLEVEL_DB" : { + "id" : 3, + "separator": ":", + "instance" : "redis" + }, + "CONFIG_DB" : { + "id" : 4, + "separator": "|", + "instance" : "redis" + }, + "PFC_WD_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "FLEX_COUNTER_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "STATE_DB" : { + "id" : 6, + "separator": "|", + "instance" : "redis" + }, + "SNMP_OVERLAY_DB" : { + "id" : 7, + "separator": "|", + "instance" : "redis" + } + }, + "VERSION" : "1.1" +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic2/config_db.json b/src/sonic-config-engine/tests/mock_tables/asic2/config_db.json new file mode 100644 index 000000000000..2c63c0851048 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic2/config_db.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic2/database_config.json b/src/sonic-config-engine/tests/mock_tables/asic2/database_config.json new file mode 100644 index 000000000000..33fdeb2d7092 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic2/database_config.json @@ -0,0 +1,57 @@ +{ + "INSTANCES": { + "redis": { + "hostname" : "127.0.0.1", + "port" : 6379, + "unix_socket_path" : "/var/run/redis/redis.sock" + } + }, + "DATABASES" : { + "APPL_DB" : { + "id" : 0, + "separator": ":", + "instance" : "redis" + }, + "ASIC_DB" : { + "id" : 1, + "separator": ":", + "instance" : "redis" + }, + "COUNTERS_DB" : { + "id" : 2, + "separator": ":", + "instance" : "redis" + }, + "LOGLEVEL_DB" : { + "id" : 3, + "separator": ":", + "instance" : "redis" + }, + "CONFIG_DB" : { + "id" : 4, + "separator": "|", + "instance" : "redis" + }, + "PFC_WD_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "FLEX_COUNTER_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "STATE_DB" : { + "id" : 6, + "separator": "|", + "instance" : "redis" + }, + "SNMP_OVERLAY_DB" : { + "id" : 7, + "separator": "|", + "instance" : "redis" + } + }, + "VERSION" : "1.1" +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic3/config_db.json b/src/sonic-config-engine/tests/mock_tables/asic3/config_db.json new file mode 100644 index 000000000000..2c63c0851048 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic3/config_db.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/sonic-config-engine/tests/mock_tables/asic3/database_config.json b/src/sonic-config-engine/tests/mock_tables/asic3/database_config.json new file mode 100644 index 000000000000..33fdeb2d7092 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/asic3/database_config.json @@ -0,0 +1,57 @@ +{ + "INSTANCES": { + "redis": { + "hostname" : "127.0.0.1", + "port" : 6379, + "unix_socket_path" : "/var/run/redis/redis.sock" + } + }, + "DATABASES" : { + "APPL_DB" : { + "id" : 0, + "separator": ":", + "instance" : "redis" + }, + "ASIC_DB" : { + "id" : 1, + "separator": ":", + "instance" : "redis" + }, + "COUNTERS_DB" : { + "id" : 2, + "separator": ":", + "instance" : "redis" + }, + "LOGLEVEL_DB" : { + "id" : 3, + "separator": ":", + "instance" : "redis" + }, + "CONFIG_DB" : { + "id" : 4, + "separator": "|", + "instance" : "redis" + }, + "PFC_WD_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "FLEX_COUNTER_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "STATE_DB" : { + "id" : 6, + "separator": "|", + "instance" : "redis" + }, + "SNMP_OVERLAY_DB" : { + "id" : 7, + "separator": "|", + "instance" : "redis" + } + }, + "VERSION" : "1.1" +} diff --git a/src/sonic-config-engine/tests/mock_tables/config_db.json b/src/sonic-config-engine/tests/mock_tables/config_db.json new file mode 100644 index 000000000000..25937ca0dae1 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/config_db.json @@ -0,0 +1,236 @@ +{ + "PORT|Ethernet0": { + "lanes": "29,30,31,32", + "description": "config_db:switch-01t1:port1", + "pfc_asym": "off", + "mtu": "9100", + "alias": "fortyGigE0/0", + "admin_status": "up", + "speed": "10000" + }, + "PORT|Ethernet4": { + "lanes": "25,26,27,28", + "description": "config_db:server1:port1", + "pfc_asym": "off", + "mtu": "9100", + "alias": "fortyGigE0/4", + "admin_status": "up", + "speed": "25000", + "mux_cable": "true" + }, + "PORT|Ethernet8": { + "lanes": "37,38,39,40", + "description": "config_db:Interface description", + "pfc_asym": "off", + "mtu": "9100", + "alias": "fortyGigE0/8", + "admin_status": "up", + "speed": "40000", + "mux_cable": "true" + }, + "PORT|Ethernet12": { + "lanes": "33,34,35,36", + "description": "config_db:Interface description", + "pfc_asym": "off", + "mtu": "9100", + "alias": "fortyGigE0/12", + "admin_status": "up", + "speed": "10000" + }, + "PORT|Ethernet16": { + "alias": "fortyGigE0/16", + "pfc_asym": "off", + "lanes": "41,42,43,44", + "description": "config_db:fortyGigE0/16", + "mtu": "9100" + }, + "PORT|Ethernet20": { + "alias": "fortyGigE0/20", + "pfc_asym": "off", + "lanes": "45,46,47,48", + "description": "config_db:fortyGigE0/20", + "mtu": "9100" + }, + "PORT|Ethernet24": { + "alias": "fortyGigE0/24", + "pfc_asym": "off", + "lanes": "5,6,7,8", + "description": "config_db:fortyGigE0/24", + "mtu": "9100" + }, + "PORT|Ethernet28": { + "alias": "fortyGigE0/28", + "pfc_asym": "off", + "lanes": "1,2,3,4", + "description": "config_db:fortyGigE0/28", + "mtu": "9100" + }, + "PORT|Ethernet32": { + "alias": "fortyGigE0/32", + "pfc_asym": "off", + "lanes": "9,10,11,12", + "description": "config_db:fortyGigE0/32", + "mtu": "9100" + }, + "PORT|Ethernet36": { + "alias": "fortyGigE0/36", + "pfc_asym": "off", + "lanes": "13,14,15,16", + "description": "config_db:fortyGigE0/36", + "mtu": "9100" + }, + "PORT|Ethernet40": { + "alias": "fortyGigE0/40", + "pfc_asym": "off", + "lanes": "21,22,23,24", + "description": "config_db:fortyGigE0/40", + "mtu": "9100" + }, + "PORT|Ethernet44": { + "alias": "fortyGigE0/44", + "pfc_asym": "off", + "lanes": "17,18,19,20", + "description": "config_db:fortyGigE0/44", + "mtu": "9100" + }, + "PORT|Ethernet48": { + "alias": "fortyGigE0/48", + "pfc_asym": "off", + "lanes": "49,50,51,52", + "description": "config_db:fortyGigE0/48", + "mtu": "9100" + }, + "PORT|Ethernet52": { + "alias": "fortyGigE0/52", + "pfc_asym": "off", + "lanes": "53,54,55,56", + "description": "config_db:fortyGigE0/52", + "mtu": "9100" + }, + "PORT|Ethernet56": { + "alias": "fortyGigE0/56", + "pfc_asym": "off", + "lanes": "61,62,63,64", + "description": "config_db:fortyGigE0/56", + "mtu": "9100" + }, + "PORT|Ethernet60": { + "alias": "fortyGigE0/60", + "pfc_asym": "off", + "lanes": "57,58,59,60", + "description": "config_db:fortyGigE0/60", + "mtu": "9100" + }, + "PORT|Ethernet64": { + "alias": "fortyGigE0/64", + "pfc_asym": "off", + "lanes": "65,66,67,68", + "description": "config_db:fortyGigE0/64", + "mtu": "9100" + }, + "PORT|Ethernet68": { + "alias": "fortyGigE0/68", + "pfc_asym": "off", + "lanes": "69,70,71,72", + "description": "config_db:fortyGigE0/68", + "mtu": "9100" + }, + "PORT|Ethernet72": { + "alias": "fortyGigE0/72", + "pfc_asym": "off", + "lanes": "77,78,79,80", + "description": "config_db:fortyGigE0/72", + "mtu": "9100" + }, + "PORT|Ethernet76": { + "alias": "fortyGigE0/76", + "pfc_asym": "off", + "lanes": "73,74,75,76", + "description": "config_db:fortyGigE0/76", + "mtu": "9100" + }, + "PORT|Ethernet80": { + "alias": "fortyGigE0/80", + "pfc_asym": "off", + "lanes": "105,106,107,108", + "description": "config_db:fortyGigE0/80", + "mtu": "9100" + }, + "PORT|Ethernet84": { + "alias": "fortyGigE0/84", + "pfc_asym": "off", + "lanes": "109,110,111,112", + "description": "config_db:fortyGigE0/84", + "mtu": "9100" + }, + "PORT|Ethernet88": { + "alias": "fortyGigE0/88", + "pfc_asym": "off", + "lanes": "117,118,119,120", + "description": "config_db:fortyGigE0/88", + "mtu": "9100" + }, + "PORT|Ethernet92": { + "alias": "fortyGigE0/92", + "pfc_asym": "off", + "lanes": "113,114,115,116", + "description": "config_db:fortyGigE0/92", + "mtu": "9100" + }, + "PORT|Ethernet96": { + "alias": "fortyGigE0/96", + "pfc_asym": "off", + "lanes": "121,122,123,124", + "description": "config_db:fortyGigE0/96", + "mtu": "9100" + }, + "PORT|Ethernet100": { + "alias": "fortyGigE0/100", + "pfc_asym": "off", + "lanes": "125,126,127,128", + "description": "config_db:fortyGigE0/100", + "mtu": "9100" + }, + "PORT|Ethernet104": { + "alias": "fortyGigE0/104", + "pfc_asym": "off", + "lanes": "85,86,87,88", + "description": "config_db:fortyGigE0/104", + "mtu": "9100" + }, + "PORT|Ethernet108": { + "alias": "fortyGigE0/108", + "pfc_asym": "off", + "lanes": "81,82,83,84", + "description": "config_db:fortyGigE0/108", + "mtu": "9100" + }, + "PORT|Ethernet112": { + "alias": "fortyGigE0/112", + "pfc_asym": "off", + "lanes": "89,90,91,92", + "description": "config_db:fortyGigE0/112", + "mtu": "9100" + }, + "PORT|Ethernet116": { + "alias": "fortyGigE0/116", + "pfc_asym": "off", + "lanes": "93,94,95,96", + "description": "config_db:fortyGigE0/116", + "mtu": "9100" + }, + "PORT|Ethernet120": { + "alias": "fortyGigE0/120", + "pfc_asym": "off", + "lanes": "97,98,99,100", + "description": "config_db:fortyGigE0/120", + "mtu": "9100" + }, + "PORT|Ethernet124": { + "alias": "fortyGigE0/124", + "pfc_asym": "off", + "lanes": "101,102,103,104", + "description": "config_db:fortyGigE0/124", + "mtu": "9100" + } +} diff --git a/src/sonic-config-engine/tests/mock_tables/database_config.json b/src/sonic-config-engine/tests/mock_tables/database_config.json new file mode 100644 index 000000000000..33fdeb2d7092 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/database_config.json @@ -0,0 +1,57 @@ +{ + "INSTANCES": { + "redis": { + "hostname" : "127.0.0.1", + "port" : 6379, + "unix_socket_path" : "/var/run/redis/redis.sock" + } + }, + "DATABASES" : { + "APPL_DB" : { + "id" : 0, + "separator": ":", + "instance" : "redis" + }, + "ASIC_DB" : { + "id" : 1, + "separator": ":", + "instance" : "redis" + }, + "COUNTERS_DB" : { + "id" : 2, + "separator": ":", + "instance" : "redis" + }, + "LOGLEVEL_DB" : { + "id" : 3, + "separator": ":", + "instance" : "redis" + }, + "CONFIG_DB" : { + "id" : 4, + "separator": "|", + "instance" : "redis" + }, + "PFC_WD_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "FLEX_COUNTER_DB" : { + "id" : 5, + "separator": ":", + "instance" : "redis" + }, + "STATE_DB" : { + "id" : 6, + "separator": "|", + "instance" : "redis" + }, + "SNMP_OVERLAY_DB" : { + "id" : 7, + "separator": "|", + "instance" : "redis" + } + }, + "VERSION" : "1.1" +} diff --git a/src/sonic-config-engine/tests/mock_tables/database_global.json b/src/sonic-config-engine/tests/mock_tables/database_global.json new file mode 100644 index 000000000000..386bcc368836 --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/database_global.json @@ -0,0 +1,24 @@ +{ + "INCLUDES" : [ + { + "include" : "database_config.json" + }, + { + "namespace" : "asic0", + "include" : "./asic0/database_config.json" + }, + { + "namespace" : "asic1", + "include" : "./asic1/database_config.json" + }, + { + "namespace" : "asic2", + "include" : "./asic2/database_config.json" + }, + { + "namespace" : "asic3", + "include" : "./asic3/database_config.json" + } + ], + "VERSION" : "1.0" +} diff --git a/src/sonic-config-engine/tests/mock_tables/dbconnector.py b/src/sonic-config-engine/tests/mock_tables/dbconnector.py new file mode 100644 index 000000000000..2b152c36a06a --- /dev/null +++ b/src/sonic-config-engine/tests/mock_tables/dbconnector.py @@ -0,0 +1,149 @@ +# MONKEY PATCH!!! +import json +import os +import sys +#from unittest import mock + +import mockredis +import redis +from sonic_py_common import multi_asic +import swsssdk +from swsssdk import SonicDBConfig, SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector +from swsscommon import swsscommon +#import swsscommon.swsscommon + + +topo = None +dedicated_dbs = {} + +def clean_up_config(): + # Set SonicDBConfig variables to initial state + # so that it can be loaded with single or multiple + # namespaces before the test begins. + SonicDBConfig._sonic_db_config = {} + SonicDBConfig._sonic_db_global_config_init = False + SonicDBConfig._sonic_db_config_init = False + +def load_namespace_config(): + # To support multi asic testing + # SonicDBConfig load_sonic_global_db_config + # is invoked to load multiple namespaces + clean_up_config() + SonicDBConfig.load_sonic_global_db_config( + global_db_file_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'database_global.json')) + +def load_database_config(): + # Load local database_config.json for single namespace test scenario + clean_up_config() + SonicDBConfig.load_sonic_db_config( + sonic_db_file_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'database_config.json')) + +_old_connect_ConfigDBConnector = ConfigDBConnector.connect + +def connect_ConfigDBConnector(self, wait_for_init=True, retry_on=False): + # add topo to kwargs for testing different topology + self.dbintf.redis_kwargs['topo'] = topo + # add the namespace to kwargs for testing multi asic + self.dbintf.redis_kwargs['namespace'] = self.namespace + self.dbintf.redis_kwargs['db_name'] = 'CONFIG_DB' + self.dbintf.redis_kwargs['decode_responses'] = True + _old_connect_ConfigDBConnector(self, wait_for_init, retry_on) + +def _subscribe_keyspace_notification(self, db_name, client): + pass + + +def config_set(self, *args): + pass + + +class MockPubSub: + def get_message(self): + return None + + def psubscribe(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return self + + def listen(self): + return [] + + def punsubscribe(self, *args, **kwargs): + pass + + def clear(self): + pass + +INPUT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class SwssSyncClient(mockredis.MockRedis): + def __init__(self, *args, **kwargs): + super(SwssSyncClient, self).__init__(strict=True, *args, **kwargs) + # Namespace is added in kwargs specifically for unit-test + # to identify the file path to load the db json files. + topo = kwargs.pop('topo') + namespace = kwargs.pop('namespace') + db_name = kwargs.pop('db_name') + self.decode_responses = kwargs.pop('decode_responses', False) == True + fname = db_name.lower() + ".json" + self.pubsub = MockPubSub() + + if namespace is not None and namespace is not multi_asic.DEFAULT_NAMESPACE: + fname = os.path.join(INPUT_DIR, namespace, fname) + elif topo is not None: + fname = os.path.join(INPUT_DIR, topo, fname) + else: + fname = os.path.join(INPUT_DIR, fname) + + if os.path.exists(fname): + with open(fname) as f: + js = json.load(f) + for k, v in js.items(): + if 'expireat' in v and 'ttl' in v and 'type' in v and 'value' in v: + # database is in redis-dump format + if v['type'] == 'hash': + # ignore other types for now since sonic has hset keys only in the db + for attr, value in v['value'].items(): + self.hset(k, attr, value) + else: + for attr, value in v.items(): + self.hset(k, attr, value) + + # Patch mockredis/mockredis/client.py + # The offical implementation assume decode_responses=False + # Here we detect the option and decode after doing encode + def _encode(self, value): + "Return a bytestring representation of the value. Taken from redis-py connection.py" + + value = super(SwssSyncClient, self)._encode(value) + + if self.decode_responses: + return value.decode('utf-8') + + # Patch mockredis/mockredis/client.py + # The official implementation will filter out keys with a slash '/' + # ref: https://github.com/locationlabs/mockredis/blob/master/mockredis/client.py + def keys(self, pattern='*'): + """Emulate keys.""" + import fnmatch + import re + + # Make regex out of glob styled pattern. + regex = fnmatch.translate(pattern) + regex = re.compile(regex) + + # Find every key that matches the pattern + return [key for key in self.redis if regex.match(key)] + +swsssdk.interface.DBInterface._subscribe_keyspace_notification = _subscribe_keyspace_notification +mockredis.MockRedis.config_set = config_set +redis.StrictRedis = SwssSyncClient +ConfigDBConnector.connect = connect_ConfigDBConnector +swsscommon.SonicV2Connector = SonicV2Connector +swsscommon.ConfigDBConnector = ConfigDBConnector +swsscommon.ConfigDBPipeConnector = ConfigDBPipeConnector diff --git a/src/sonic-config-engine/tests/test_cfggen.py b/src/sonic-config-engine/tests/test_cfggen.py index 79a3fd6fd9ac..5702841ceea4 100644 --- a/src/sonic-config-engine/tests/test_cfggen.py +++ b/src/sonic-config-engine/tests/test_cfggen.py @@ -32,8 +32,11 @@ def setUp(self): self.ecmp_graph = os.path.join(self.test_dir, 'fg-ecmp-sample-minigraph.xml') self.sample_resource_graph = os.path.join(self.test_dir, 'sample-graph-resource-type.xml') self.sample_subintf_graph = os.path.join(self.test_dir, 'sample-graph-subintf.xml') + # To ensure that mock config_db data is used for unit-test cases + os.environ["CFGGEN_UNIT_TESTING"] = "2" def tearDown(self): + os.environ["CFGGEN_UNIT_TESTING"] = "" try: os.remove(self.output_file) os.remove(self.output2_file) @@ -546,8 +549,51 @@ def test_minigraph_neighbor_interfaces(self): ) ) + def test_minigraph_neighbor_interfaces_config_db(self): + # test to check if PORT table is retrieved from config_db + argument = '-m "' + self.sample_graph_simple_case + '" -v "PORT"' + output = self.run_script(argument) + + self.assertEqual( + utils.to_dict(output.strip()), + utils.to_dict( + "{'Ethernet0': {'lanes': '29,30,31,32', 'description': 'config_db:switch-01t1:port1', 'pfc_asym': 'off', 'mtu': '9100', 'alias': 'fortyGigE0/0', 'admin_status': 'up', 'speed': '10000', 'autoneg': '1'}, " + "'Ethernet4': {'lanes': '25,26,27,28', 'description': 'config_db:server1:port1', 'pfc_asym': 'off', 'mtu': '9100', 'alias': 'fortyGigE0/4', 'admin_status': 'up', 'speed': '25000', 'autoneg': '1', 'mux_cable': 'true'}, " + "'Ethernet8': {'lanes': '37,38,39,40', 'description': 'Interface description', 'pfc_asym': 'off', 'mtu': '9100', 'alias': 'fortyGigE0/8', 'admin_status': 'up', 'speed': '40000', 'autoneg': '1', 'mux_cable': 'true'}, " + "'Ethernet12': {'lanes': '33,34,35,36', 'description': 'Interface description', 'pfc_asym': 'off', 'mtu': '9100', 'alias': 'fortyGigE0/12', 'admin_status': 'up', 'speed': '10000', 'autoneg': '1'}, " + "'Ethernet16': {'alias': 'fortyGigE0/16', 'pfc_asym': 'off', 'lanes': '41,42,43,44', 'description': 'config_db:fortyGigE0/16', 'mtu': '9100'}, " + "'Ethernet20': {'alias': 'fortyGigE0/20', 'pfc_asym': 'off', 'lanes': '45,46,47,48', 'description': 'config_db:fortyGigE0/20', 'mtu': '9100'}, " + "'Ethernet24': {'alias': 'fortyGigE0/24', 'pfc_asym': 'off', 'lanes': '5,6,7,8', 'description': 'config_db:fortyGigE0/24', 'mtu': '9100'}, " + "'Ethernet28': {'alias': 'fortyGigE0/28', 'pfc_asym': 'off', 'lanes': '1,2,3,4', 'description': 'config_db:fortyGigE0/28', 'mtu': '9100'}, " + "'Ethernet32': {'alias': 'fortyGigE0/32', 'pfc_asym': 'off', 'lanes': '9,10,11,12', 'description': 'config_db:fortyGigE0/32', 'mtu': '9100'}, " + "'Ethernet36': {'alias': 'fortyGigE0/36', 'pfc_asym': 'off', 'lanes': '13,14,15,16', 'description': 'config_db:fortyGigE0/36', 'mtu': '9100'}, " + "'Ethernet40': {'alias': 'fortyGigE0/40', 'pfc_asym': 'off', 'lanes': '21,22,23,24', 'description': 'config_db:fortyGigE0/40', 'mtu': '9100'}, " + "'Ethernet44': {'alias': 'fortyGigE0/44', 'pfc_asym': 'off', 'lanes': '17,18,19,20', 'description': 'config_db:fortyGigE0/44', 'mtu': '9100'}, " + "'Ethernet48': {'alias': 'fortyGigE0/48', 'pfc_asym': 'off', 'lanes': '49,50,51,52', 'description': 'config_db:fortyGigE0/48', 'mtu': '9100'}, " + "'Ethernet52': {'alias': 'fortyGigE0/52', 'pfc_asym': 'off', 'lanes': '53,54,55,56', 'description': 'config_db:fortyGigE0/52', 'mtu': '9100'}, " + "'Ethernet56': {'alias': 'fortyGigE0/56', 'pfc_asym': 'off', 'lanes': '61,62,63,64', 'description': 'config_db:fortyGigE0/56', 'mtu': '9100'}, " + "'Ethernet60': {'alias': 'fortyGigE0/60', 'pfc_asym': 'off', 'lanes': '57,58,59,60', 'description': 'config_db:fortyGigE0/60', 'mtu': '9100'}, " + "'Ethernet64': {'alias': 'fortyGigE0/64', 'pfc_asym': 'off', 'lanes': '65,66,67,68', 'description': 'config_db:fortyGigE0/64', 'mtu': '9100'}, " + "'Ethernet68': {'alias': 'fortyGigE0/68', 'pfc_asym': 'off', 'lanes': '69,70,71,72', 'description': 'config_db:fortyGigE0/68', 'mtu': '9100'}, " + "'Ethernet72': {'alias': 'fortyGigE0/72', 'pfc_asym': 'off', 'lanes': '77,78,79,80', 'description': 'config_db:fortyGigE0/72', 'mtu': '9100'}, " + "'Ethernet76': {'alias': 'fortyGigE0/76', 'pfc_asym': 'off', 'lanes': '73,74,75,76', 'description': 'config_db:fortyGigE0/76', 'mtu': '9100'}, " + "'Ethernet80': {'alias': 'fortyGigE0/80', 'pfc_asym': 'off', 'lanes': '105,106,107,108', 'description': 'config_db:fortyGigE0/80', 'mtu': '9100'}, " + "'Ethernet84': {'alias': 'fortyGigE0/84', 'pfc_asym': 'off', 'lanes': '109,110,111,112', 'description': 'config_db:fortyGigE0/84', 'mtu': '9100'}, " + "'Ethernet88': {'alias': 'fortyGigE0/88', 'pfc_asym': 'off', 'lanes': '117,118,119,120', 'description': 'config_db:fortyGigE0/88', 'mtu': '9100'}, " + "'Ethernet92': {'alias': 'fortyGigE0/92', 'pfc_asym': 'off', 'lanes': '113,114,115,116', 'description': 'config_db:fortyGigE0/92', 'mtu': '9100'}, " + "'Ethernet96': {'alias': 'fortyGigE0/96', 'pfc_asym': 'off', 'lanes': '121,122,123,124', 'description': 'config_db:fortyGigE0/96', 'mtu': '9100'}, " + "'Ethernet100': {'alias': 'fortyGigE0/100', 'pfc_asym': 'off', 'lanes': '125,126,127,128', 'description': 'config_db:fortyGigE0/100', 'mtu': '9100'}, " + "'Ethernet104': {'alias': 'fortyGigE0/104', 'pfc_asym': 'off', 'lanes': '85,86,87,88', 'description': 'config_db:fortyGigE0/104', 'mtu': '9100'}, " + "'Ethernet108': {'alias': 'fortyGigE0/108', 'pfc_asym': 'off', 'lanes': '81,82,83,84', 'description': 'config_db:fortyGigE0/108', 'mtu': '9100'}, " + "'Ethernet112': {'alias': 'fortyGigE0/112', 'pfc_asym': 'off', 'lanes': '89,90,91,92', 'description': 'config_db:fortyGigE0/112', 'mtu': '9100'}, " + "'Ethernet116': {'alias': 'fortyGigE0/116', 'pfc_asym': 'off', 'lanes': '93,94,95,96', 'description': 'config_db:fortyGigE0/116', 'mtu': '9100'}, " + "'Ethernet120': {'alias': 'fortyGigE0/120', 'pfc_asym': 'off', 'lanes': '97,98,99,100', 'description': 'config_db:fortyGigE0/120', 'mtu': '9100'}, " + "'Ethernet124': {'alias': 'fortyGigE0/124', 'pfc_asym': 'off', 'lanes': '101,102,103,104', 'description': 'config_db:fortyGigE0/124', 'mtu': '9100'}}" + ) + ) + def test_minigraph_extra_ethernet_interfaces(self, **kwargs): - graph_file = kwargs.get('graph_file', self.sample_graph_simple) + graph_file = kwargs.get('graph_file', self.sample_graph_simple) argument = '-m "' + graph_file + '" -p "' + self.port_config + '" -v "PORT"' output = self.run_script(argument) diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index 66b8f2bcdaff..3240f2dcfb2a 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -30,6 +30,7 @@ def setUp(self): self.radv_test_minigraph = os.path.join(self.test_dir, 'radv-test-sample-graph.xml') self.dell9332_t1_minigraph = os.path.join(self.test_dir, 'sample-dell-9332-t1-minigraph.xml') self.output_file = os.path.join(self.test_dir, 'output') + os.environ["CFGGEN_UNIT_TESTING"] = "2" def run_script(self, argument): print('CMD: sonic-cfggen ' + argument) @@ -399,6 +400,7 @@ def test_ntp_conf(self): assert utils.cmp(expected, self.output_file), self.run_diff(expected, self.output_file) def tearDown(self): + os.environ["CFGGEN_UNIT_TESTING"] = "" try: os.remove(self.output_file) except OSError: diff --git a/src/sonic-config-engine/tests/test_multinpu_cfggen.py b/src/sonic-config-engine/tests/test_multinpu_cfggen.py index ab952515f96c..8c4e097ebff3 100644 --- a/src/sonic-config-engine/tests/test_multinpu_cfggen.py +++ b/src/sonic-config-engine/tests/test_multinpu_cfggen.py @@ -29,6 +29,7 @@ def setUp(self): for asic in range(NUM_ASIC): self.port_config.append(os.path.join(self.test_data_dir, "sample_port_config-{}.ini".format(asic))) self.output_file = os.path.join(self.test_dir, 'output') + os.environ["CFGGEN_UNIT_TESTING"] = "2" def run_script(self, argument, check_stderr=False): print('\n Running sonic-cfggen ' + argument) @@ -213,6 +214,19 @@ def test_frontend_asic_ports(self): "Ethernet-BP8": { "admin_status": "up", "alias": "Eth6-ASIC0", "asic_port_name": "Eth6-ASIC0", "description": "ASIC3:Eth0-ASIC3", "index": "2", "lanes": "21,22,23,24", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }, "Ethernet-BP12": { "admin_status": "up", "alias": "Eth7-ASIC0", "asic_port_name": "Eth7-ASIC0", "description": "ASIC3:Eth1-ASIC3", "index": "3", "lanes": "25,26,27,28", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }}) + def test_frontend_asic_ports_config_db(self): + argument = "-m {} -n asic0 --var-json \"PORT\"".format(self.sample_graph) + output = json.loads(self.run_script(argument)) + self.assertDictEqual(output, + {"Ethernet0": { "admin_status": "up", "alias": "Ethernet1/1", "asic_port_name": "Eth0-ASIC0", "description": "01T2:Ethernet1:config_db", "index": "0", "lanes": "33,34,35,36", "mtu": "9100", "pfc_asym": "off", "role": "Ext", "speed": "40000", "autoneg": "1" }, + "Ethernet4": { "admin_status": "up", "alias": "Ethernet1/2", "asic_port_name": "Eth1-ASIC0", "description": "01T2:Ethernet2:config_db", "index": "1", "lanes": "29,30,31,32", "mtu": "9100", "pfc_asym": "off", "role": "Ext", "speed": "40000", "autoneg": "1" }, + "Ethernet8": { "admin_status": "up", "alias": "Ethernet1/3", "asic_port_name": "Eth2-ASIC0", "description": "Ethernet1/3:config_db", "index": "2", "lanes": "41,42,43,44", "mtu": "9100", "pfc_asym": "off", "role": "Ext", "speed": "40000" }, + "Ethernet12": { "admin_status": "up", "alias": "Ethernet1/4", "asic_port_name": "Eth3-ASIC0", "description": "Ethernet1/4:config_db", "index": "3", "lanes": "37,38,39,40", "mtu": "9100", "pfc_asym": "off", "role": "Ext", "speed": "40000" }, + "Ethernet-BP0": { "admin_status": "up", "alias": "Eth4-ASIC0", "asic_port_name": "Eth4-ASIC0", "description": "ASIC2:Eth0-ASIC2:config_db", "index": "0", "lanes": "13,14,15,16", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }, + "Ethernet-BP4": { "admin_status": "up", "alias": "Eth5-ASIC0", "asic_port_name": "Eth5-ASIC0", "description": "ASIC2:Eth1-ASIC2:config_db", "index": "1", "lanes": "17,18,19,20", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }, + "Ethernet-BP8": { "admin_status": "up", "alias": "Eth6-ASIC0", "asic_port_name": "Eth6-ASIC0", "description": "ASIC3:Eth0-ASIC3:config_db", "index": "2", "lanes": "21,22,23,24", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }, + "Ethernet-BP12": { "admin_status": "up", "alias": "Eth7-ASIC0", "asic_port_name": "Eth7-ASIC0", "description": "ASIC3:Eth1-ASIC3:config_db", "index": "3", "lanes": "25,26,27,28", "mtu": "9100", "pfc_asym": "off", "role": "Int", "speed": "40000" }}) + def test_frontend_asic_device_neigh(self): argument = "-m {} -p {} -n asic0 --var-json \"DEVICE_NEIGHBOR\"".format(self.sample_graph, self.port_config[0]) output = json.loads(self.run_script(argument)) @@ -409,3 +423,6 @@ def test_bgpd_frr_frontendasic(self): def test_bgpd_frr_backendasic(self): self.assertTrue(*self.run_frr_asic_case('bgpd/bgpd.conf.j2', 'bgpd_frr_backend_asic.conf', "asic3", self.port_config[3])) + + def tearDown(self): + os.environ["CFGGEN_UNIT_TESTING"] = ""