Merge branch 'master' of https://github.com/suresh-rupanagudi/sonic-s…

…nmpagent into snmp_trap_infra

azure-pipelines.yml

@@ -19,7 +19,9 @@ stages:
   - job:
     displayName: "build"
     timeoutInMinutes: 60
-
+    variables:
+      DIFF_COVER_CHECK_THRESHOLD: 50
+      DIFF_COVER_ENABLE: 'true'
     pool:
       vmImage: ubuntu-20.04
 
@@ -44,10 +46,10 @@ stages:
 
     - script: |
         set -ex
-
+        sudo apt-get -y purge libhiredis-dev libnl-3-dev libnl-route-3-dev
         sudo dpkg -i ../target/debs/buster/{libswsscommon_1.0.0_amd64.deb,python3-swsscommon_1.0.0_amd64.deb,libnl-3-200_*.deb,libnl-genl-3-200_*.deb,libnl-nf-3-200_*.deb,libnl-route-3-200_*.deb,libhiredis0.14_*.deb}
-        sudo python3 -m pip install ../target/python-wheels/swsssdk*-py3-*.whl
-        sudo python3 -m pip install ../target/python-wheels/sonic_py_common-1.0-py3-none-any.whl
+        sudo python3 -m pip install ../target/python-wheels/buster/swsssdk*-py3-*.whl
+        sudo python3 -m pip install ../target/python-wheels/buster/sonic_py_common-1.0-py3-none-any.whl
         python3 setup.py bdist_wheel
         cp dist/*.whl $(Build.ArtifactStagingDirectory)/
       displayName: "Build"
@@ -59,6 +61,15 @@ stages:
         python3 setup.py test
       displayName: "Unit tests"
 
+    - script: |
+        set -ex
+        # Install .NET CORE
+        curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+        sudo apt-add-repository https://packages.microsoft.com/debian/10/prod
+        sudo apt-get update
+        sudo apt-get install -y dotnet-sdk-5.0
+      displayName: "Install .NET CORE"
+
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'

src/sonic_ax_impl/main.py

@@ -9,7 +9,7 @@
 import sys
 
 import ax_interface
-from sonic_ax_impl.mibs import ieee802_1ab
+from sonic_ax_impl.mibs import ieee802_1ab, Namespace
 from . import logger
 from .mibs.ietf import rfc1213, rfc2737, rfc2863, rfc3433, rfc4292, rfc4363, link_up_down_trap
 from .mibs.vendor import dell, cisco, broadcom
@@ -60,6 +60,8 @@ def main(update_frequency=None):
     global event_loop
 
     try:
+        Namespace.init_sonic_db_config()
+
         # initialize handler and set update frequency (or use the default)
         agent = ax_interface.Agent(SonicMIB, update_frequency or DEFAULT_UPDATE_FREQUENCY, event_loop, trap_handlers)
 

src/sonic_ax_impl/mibs/__init__.py

@@ -44,7 +44,6 @@
 
 redis_kwargs = {'unix_socket_path': '/var/run/redis/redis.sock'}
 
-
 def get_neigh_info(neigh_key):
     """
     split neigh_key string of the format:
@@ -219,16 +218,11 @@ def init_db():
     Connects to DB
     :return: db_conn
     """
-    if not SonicDBConfig.isInit():
-        if multi_asic.is_multi_asic():
-            # Load the global config file database_global.json once.
-            SonicDBConfig.load_sonic_global_db_config()
-        else:
-            SonicDBConfig.load_sonic_db_config()
+    Namespace.init_sonic_db_config()
+
     # SyncD database connector. THIS MUST BE INITIALIZED ON A PER-THREAD BASIS.
     # Redis PubSub objects (such as those within swsssdk) are NOT thread-safe.
     db_conn = SonicV2Connector(**redis_kwargs)
-
     return db_conn
 
 def init_mgmt_interface_tables(db_conn):
@@ -271,19 +265,21 @@ def init_sync_d_interface_tables(db_conn):
 
     # { if_name (SONiC) -> sai_id }
     # ex: { "Ethernet76" : "1000000000023" }
-    if_name_map_util, if_id_map_util = port_util.get_interface_oid_map(db_conn)
+    if_name_map_util, if_id_map_util = port_util.get_interface_oid_map(db_conn, blocking=False)
     for if_name, sai_id in if_name_map_util.items():
         if_name_str = if_name
         if (re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name_str) or \
-                re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name_str)):
+                re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name_str) or \
+                re.match(port_util.SONIC_ETHERNET_IB_RE_PATTERN, if_name_str)):
             if_name_map[if_name] = sai_id
     # As sai_id is not unique in multi-asic platform, concatenate it with
     # namespace to get a unique key. Assuming that ':' is not present in namespace
     # string or in sai id.
     # sai_id_key = namespace : sai_id
     for sai_id, if_name in if_id_map_util.items():
         if (re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name) or \
-                re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name)):
+                re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name) or \
+                re.match(port_util.SONIC_ETHERNET_IB_RE_PATTERN, if_name)):
             if_id_map[get_sai_id_key(db_conn.namespace, sai_id)] = if_name
     logger.debug("Port name map:\n" + pprint.pformat(if_name_map, indent=2))
     logger.debug("Interface name map:\n" + pprint.pformat(if_id_map, indent=2))
@@ -297,12 +293,8 @@ def init_sync_d_interface_tables(db_conn):
 
     # SyncD consistency checks.
     if not oid_name_map:
-        # In the event no interface exists that follows the SONiC pattern, no OIDs are able to be registered.
-        # A RuntimeError here will prevent the 'main' module from loading. (This is desirable.)
-        message = "No interfaces found matching pattern '{}'. SyncD database is incoherent." \
-            .format(port_util.SONIC_ETHERNET_RE_PATTERN)
-        logger.error(message)
-        raise RuntimeError(message)
+        logger.debug("There are no ports in counters DB")
+        return {}, {}, {}, {}
     elif len(if_id_map) < len(if_name_map) or len(oid_name_map) < len(if_name_map):
         # a length mismatch indicates a bad interface name
         logger.warning("SyncD database contains incoherent interface names. Interfaces must match pattern '{}'"
@@ -424,7 +416,7 @@ def init_sync_d_queue_tables(db_conn):
 
     # { Port name : Queue index (SONiC) -> sai_id }
     # ex: { "Ethernet0:2" : "1000000000023" }
-    queue_name_map = db_conn.get_all(COUNTERS_DB, COUNTERS_QUEUE_NAME_MAP, blocking=True)
+    queue_name_map = db_conn.get_all(COUNTERS_DB, COUNTERS_QUEUE_NAME_MAP, blocking=False)
     logger.debug("Queue name map:\n" + pprint.pformat(queue_name_map, indent=2))
 
     # Parse the queue_name_map and create the following maps:
@@ -455,13 +447,11 @@ def init_sync_d_queue_tables(db_conn):
 
     # SyncD consistency checks.
     if not port_queues_map:
-        # In the event no queue exists that follows the SONiC pattern, no OIDs are able to be registered.
-        # A RuntimeError here will prevent the 'main' module from loading. (This is desirable.)
-        logger.error("No queues found in the Counter DB. SyncD database is incoherent.")
-        raise RuntimeError('The port_queues_map is not defined')
-    elif not queue_stat_map:
-        logger.error("No queue stat counters found in the Counter DB. SyncD database is incoherent.")
-        raise RuntimeError('The queue_stat_map is not defined')
+        logger.debug("Counters DB does not contain ports")
+        return {}, {}, {}
+    if not queue_stat_map:
+        logger.debug("No queue stat counters found in the Counter DB.")
+        return {}, {}, {}
 
     for queues in port_queue_list_map.values():
         queues.sort()
@@ -541,14 +531,32 @@ def get_oidvalue(self, oid):
         return self.oid_map[oid]
 
 class Namespace:
+
+    """
+        Sonic database initialized flag.
+    """
+    db_config_loaded = False
+
+    @staticmethod
+    def init_sonic_db_config():
+        """
+        Initialize SonicDBConfig
+        """
+        if Namespace.db_config_loaded:
+            return
+
+        if multi_asic.is_multi_asic():
+            # Load the global config file database_global.json once.
+            SonicDBConfig.load_sonic_global_db_config()
+        else:
+            SonicDBConfig.load_sonic_db_config()
+
+        Namespace.db_config_loaded = True
+
     @staticmethod
     def init_namespace_dbs():
         db_conn = []
-        if not SonicDBConfig.isInit():
-            if multi_asic.is_multi_asic():
-                SonicDBConfig.load_sonic_global_db_config()
-            else:
-                SonicDBConfig.load_sonic_db_config()
+        Namespace.init_sonic_db_config()
         host_namespace_idx = 0
         for idx, namespace in enumerate(SonicDBConfig.get_ns_list()): 
             if namespace == multi_asic.DEFAULT_NAMESPACE:

src/sonic_ax_impl/mibs/ieee802_1ab.py

@@ -96,6 +96,18 @@ def poll_lldp_entry_updates(pubsub):
         return ret
     return data, interface, if_index
 
+def get_latest_notification(pubsub):
+    """
+    Fetches the latest notification recorded on a lldp entry.
+    """
+    latest_update_map = {}
+    while True:
+        data, interface, if_index = poll_lldp_entry_updates(pubsub)
+        if not data:
+            break
+        latest_update_map[interface] = (data, if_index)
+    return latest_update_map
+
 def parse_sys_capability(sys_cap):
     return bytearray([int (x, 16) for x in sys_cap.split()])
 
@@ -184,7 +196,7 @@ def reinit_data(self):
             self.if_range.append((if_oid, ))
         self.if_range.sort()
         if not self.loc_port_data:
-            logger.warning("0 - b'PORT_TABLE' is empty. No local port information could be retrieved.")
+            logger.debug("0 - b'PORT_TABLE' is empty. No local port information could be retrieved.")
 
     def _get_if_entry(self, if_name):
         if_table = ""
@@ -542,19 +554,17 @@ def _update_per_namespace_data(self, pubsub):
         """
         Listen to updates in APP DB, update local cache
         """
-        while True:
-            data, interface, if_index = poll_lldp_entry_updates(pubsub)
-
-            if not data:
-                break
-
+        event_cache = get_latest_notification(pubsub)
+        for interface in event_cache:
+            data = event_cache[interface][0]
+            if_index = event_cache[interface][1]
+
             if "set" in data:
                 self.update_rem_if_mgmt(if_index, interface)
             elif "del" in data:
-                # some remote data about that neighbor is gone, del it and try to query again
+                # if del is the latest notification, then delete it from the local cache
                 self.if_range = [sub_oid for sub_oid in self.if_range if sub_oid[0] != if_index]
-                self.update_rem_if_mgmt(if_index, interface)
-
+
     def update_data(self):
         for i in range(len(self.db_conn)):
             if not self.pubsub[i]:

src/sonic_ax_impl/mibs/ietf/rfc1213.py

@@ -79,7 +79,7 @@ def _update_from_db(self):
             neigh_str = neigh_key
             db_index = self.neigh_key_list[neigh_key]
             neigh_info = self.db_conn[db_index].get_all(mibs.APPL_DB, neigh_key, blocking=False)
-            if neigh_info is None:
+            if not neigh_info:
                 continue
             ip_family = neigh_info['family']
             if ip_family == "IPv4":
@@ -235,6 +235,11 @@ def reinit_data(self):
         self.rif_port_map, \
         self.port_rif_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_rif_tables, self.db_conn)
 
+        self.lag_name_if_name_map, \
+        self.if_name_lag_name_map, \
+        self.oid_lag_name_map, \
+        self.lag_sai_map, self.sai_lag_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_lag_tables, self.db_conn)
+
     def update_data(self):
         """
         Update redis (caches config)
@@ -246,11 +251,6 @@ def update_data(self):
 
         self.aggregate_counters()
 
-        self.lag_name_if_name_map, \
-        self.if_name_lag_name_map, \
-        self.oid_lag_name_map, \
-        self.lag_sai_map, self.sai_lag_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_lag_tables, self.db_conn)
-
         self.if_range = sorted(list(self.oid_name_map.keys()) +
                                list(self.oid_lag_name_map.keys()) +
                                list(self.mgmt_oid_name_map.keys()) +
@@ -262,8 +262,9 @@ def update_if_counters(self):
             namespace, sai_id = mibs.split_sai_id_key(sai_id_key)
             if_idx = mibs.get_index_from_str(self.if_id_map[sai_id_key])
             counters_db_data = self.namespace_db_map[namespace].get_all(mibs.COUNTERS_DB,
-                                                                        mibs.counter_table(sai_id),
-                                                                        blocking=True)
+                                                                        mibs.counter_table(sai_id))
+            if counters_db_data is None:
+                counters_db_data = {}
             self.if_counters[if_idx] = {
                 counter: int(value) for counter, value in counters_db_data.items()
             }
@@ -272,8 +273,9 @@ def update_rif_counters(self):
         rif_sai_ids = list(self.rif_port_map) + list(self.vlan_name_map)
         for sai_id in rif_sai_ids:
             counters_db_data = Namespace.dbs_get_all(self.db_conn, mibs.COUNTERS_DB,
-                                                     mibs.counter_table(mibs.split_sai_id_key(sai_id)[1]),
-                                                     blocking=False)
+                                                     mibs.counter_table(mibs.split_sai_id_key(sai_id)[1]))
+            if counters_db_data is None:
+                counters_db_data = {}
             self.rif_counters[sai_id] = {
                 counter: int(value) for counter, value in counters_db_data.items()
             }
@@ -358,8 +360,8 @@ def aggregate_counters(self):
                 port_idx = mibs.get_index_from_str(self.if_id_map[port_sai_id])
                 for port_counter_name, rif_counter_name in mibs.RIF_DROPS_AGGR_MAP.items():
                     self.if_counters[port_idx][port_counter_name] = \
-                    self.if_counters[port_idx][port_counter_name] + \
-                    self.rif_counters[rif_sai_id][rif_counter_name]
+                    self.if_counters[port_idx].get(port_counter_name, 0) + \
+                    self.rif_counters[rif_sai_id].get(rif_counter_name, 0)
 
         for vlan_sai_id, vlan_name in self.vlan_name_map.items():
             for port_counter_name, rif_counter_name in mibs.RIF_COUNTERS_AGGR_MAP.items():
@@ -408,7 +410,7 @@ def get_counter(self, sub_id, table_name):
             # self.lag_sai_map['PortChannel01'] = '2000000000006'
             # self.port_rif_map['2000000000006'] = '6000000000006'
             sai_lag_id = self.lag_sai_map[self.oid_lag_name_map[oid]]
-            sai_lag_rif_id = self.port_rif_map[sai_lag_id]
+            sai_lag_rif_id = self.port_rif_map[sai_lag_id] if sai_lag_id in self.port_rif_map else None
             if sai_lag_rif_id in self.rif_port_map:
                 # Extract the 'name' part of 'table_name'.
                 # Example: 

src/sonic_ax_impl/mibs/ietf/rfc2737.py

@@ -96,7 +96,7 @@ class XcvrInfoDB(str, Enum):
     Transceiver info keys
     """
     TYPE              = "type"
-    HARDWARE_REVISION = "hardware_rev"
+    VENDOR_REVISION   = "vendor_rev"
     SERIAL_NUMBER     = "serial"
     MANUFACTURE_NAME  = "manufacturer"
     MODEL_NAME        = "model"

src/sonic_ax_impl/mibs/ietf/rfc2863.py

@@ -93,7 +93,6 @@ def __init__(self):
         self.mgmt_alias_map = {}
         self.vlan_oid_name_map = {}
         self.vlan_name_map = {}
-        self.rif_port_map = {}
         self.if_counters = {}
         self.if_range = []
         self.if_name_map = {}
@@ -128,9 +127,6 @@ def reinit_data(self):
         self.vlan_oid_sai_map, \
         self.vlan_oid_name_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_vlan_tables, self.db_conn)
 
-        self.rif_port_map, \
-        self.port_rif_map = Namespace.get_sync_d_from_all_namespace(mibs.init_sync_d_rif_tables, self.db_conn)
-
         self.if_range = sorted(list(self.oid_name_map.keys()) +
                                list(self.oid_lag_name_map.keys()) +
                                list(self.mgmt_oid_name_map.keys()) +
@@ -145,8 +141,11 @@ def update_data(self):
         for sai_id_key in self.if_id_map:
             namespace, sai_id = mibs.split_sai_id_key(sai_id_key)
             if_idx = mibs.get_index_from_str(self.if_id_map[sai_id_key])
-            self.if_counters[if_idx] = self.namespace_db_map[namespace].get_all(mibs.COUNTERS_DB, \
-                    mibs.counter_table(sai_id), blocking=True)
+            counter_table = self.namespace_db_map[namespace].get_all(mibs.COUNTERS_DB, \
+                    mibs.counter_table(sai_id))
+            if counter_table is None:
+                counter_table = {}
+            self.if_counters[if_idx] = counter_table
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
@@ -210,18 +209,8 @@ def interface_alias(self, sub_id):
         if not entry:
             return
 
+        # This returns empty values for LAG, vlan & mgmt, which is the expected result
         result = entry.get("description", "")
-
-        if not result:
-            #RFC2863 tables don't have descriptions for LAG, vlan & mgmt; take from RFC1213
-            oid = self.get_oid(sub_id)
-            if oid in self.oid_lag_name_map:
-                result = self.oid_lag_name_map[oid]
-            elif oid in self.mgmt_oid_name_map:
-                result = self.mgmt_alias_map[self.mgmt_oid_name_map[oid]]
-            elif oid in self.vlan_oid_name_map:
-                result = self.vlan_oid_name_map[oid]
-
         return result
 
     def get_counter32(self, sub_id, table_name):

src/sonic_ax_impl/mibs/ietf/rfc4292.py

@@ -65,7 +65,7 @@ def update_data(self):
                 continue
             port_table = multi_asic.get_port_table_for_asic(db_conn.namespace)
             ent = db_conn.get_all(mibs.APPL_DB, route_str, blocking=False)
-            if ent is None:
+            if not ent:
                 continue
             nexthops = ent["nexthop"]
             ifnames = ent["ifname"]