Implementation of System ports initialization, Interface & Neighbor S…

…ynchronization... (sonic-net#1431)

This PR includes implementation of System ports initialization & orchestration, Interface & Neighbor Synchronization using GLOBAL_APP_DB, Inband Interface Configuration.

HLD: sonic-net/SONiC#639
Dependency PR's
(sonic-swss-common): sonic-net/sonic-swss-common#380
(sonic-sairedis): sonic-net/sonic-sairedis#657

* [voq] Redis instance for global app db
* [voq] System port init and orchestration
* [voq] Router interface for system ports, global app db sync
* [voq] Inband interface config
* [voq] System port neighbors

Kernel programming of static neigh and full mask static route for
system neighbors is moved to nbrmgr since orchagent should not own
kernel objects to avoid complication in warmboot reconciliation
The orchagent programs only SAI object for the system neighbors
and signals the completion of SAI programming by setting mac address
in SYSTE_NEIGH table of STATE_DB. Nbrmgr subscribes to these state
entries to program kernel entries corresponding to the system neighs

* [swss]VS tests for VOQ system ports

The following tests are added as part of virtual chassis tests
- VOQ switch objects verification
- System port object syncing with chassis app db by checking presence of
system port RIFs in chassis app db in supervisor card
- Creation of system port RIF record creation in ASIC_DB
Following changes are done for the above tests:
- SYSTEM_PORT table is added in default_config.json of line cards. These
are loaded as part of config_db loading
- Core and Port index mapping file is added in line card directories.

- use dvs_database.py lib to connect to redis databases instead of using raw connection via swsscommon DBConnectors.
- call dvs apis to get access to databases for the local redis instances. 

- Default handling for speed of the system port is removed in addSystemPorts()
- Comment added to clarify types of rif-s synced to chassis app db
- vs tests: connection to chassis app dp uses chassis app db id defined
in dvs lib instead of using the id from swsscommon

Using the chassis app db id from swsscommon instead of chassis app db id from dvd lib . This is to avoid python
lgtm alert and test failure.

Co-authored-by: vedganes <[email protected]>
Co-authored-by: vganesan-nokia <[email protected]>

cfgmgr/intfmgr.cpp

@@ -707,6 +707,8 @@ void IntfMgr::doTask(Consumer &consumer)
     SWSS_LOG_ENTER();
     static bool replayDone = false;
 
+    string table_name = consumer.getTableName();
+
     auto it = consumer.m_toSync.begin();
     while (it != consumer.m_toSync.end())
     {
@@ -718,6 +720,16 @@ void IntfMgr::doTask(Consumer &consumer)
 
         if (keys.size() == 1)
         {
+            if((table_name == CFG_VOQ_INBAND_INTERFACE_TABLE_NAME) &&
+                    (op == SET_COMMAND))
+            {
+                //No further processing needed. Just relay to orchagent
+                m_appIntfTableProducer.set(keys[0], data);
+                m_stateIntfTable.hset(keys[0], "vrf", "");
+
+                it = consumer.m_toSync.erase(it);
+                continue;
+            }
             if (!doIntfGeneralTask(keys, data, op))
             {
                 it++;

cfgmgr/intfmgrd.cpp

@@ -47,6 +47,7 @@ int main(int argc, char **argv)
             CFG_VLAN_INTF_TABLE_NAME,
             CFG_LOOPBACK_INTERFACE_TABLE_NAME,
             CFG_VLAN_SUB_INTF_TABLE_NAME,
+            CFG_VOQ_INBAND_INTERFACE_TABLE_NAME,
         };
 
         DBConnector cfgDb("CONFIG_DB", 0);

cfgmgr/nbrmgr.cpp

@@ -11,6 +11,7 @@
 #include "nbrmgr.h"
 #include "exec.h"
 #include "shellcmd.h"
+#include "subscriberstatetable.h"
 
 using namespace swss;
 
@@ -64,6 +65,20 @@ NbrMgr::NbrMgr(DBConnector *cfgDb, DBConnector *appDb, DBConnector *stateDb, con
                               TableConsumable::DEFAULT_POP_BATCH_SIZE, default_orch_pri);
     auto consumer = new Consumer(consumerStateTable, this, APP_NEIGH_RESOLVE_TABLE_NAME);
     Orch::addExecutor(consumer);
+
+    string swtype;
+    Table cfgDeviceMetaDataTable(cfgDb, CFG_DEVICE_METADATA_TABLE_NAME);
+    if(cfgDeviceMetaDataTable.hget("localhost", "switch_type", swtype))
+    {
+        //If this is voq system, let the neighbor manager subscribe to state of SYSTEM_NEIGH
+        //entries. This is used to program static neigh and static route in kernel for remote neighbors.
+        if(swtype == "voq")
+        {
+            string tableName = STATE_SYSTEM_NEIGH_TABLE_NAME;
+            Orch::addExecutor(new Consumer(new SubscriberStateTable(stateDb, tableName, TableConsumable::DEFAULT_POP_BATCH_SIZE, 0), this, tableName));
+            m_cfgVoqInbandInterfaceTable = unique_ptr<Table>(new Table(cfgDb, CFG_VOQ_INBAND_INTERFACE_TABLE_NAME));
+        }
+    }
 }
 
 bool NbrMgr::isIntfStateOk(const string &alias)
@@ -294,8 +309,253 @@ void NbrMgr::doTask(Consumer &consumer)
     } else if (table_name == APP_NEIGH_RESOLVE_TABLE_NAME)
     {
         doResolveNeighTask(consumer);
-    } else
+    } else if(table_name == STATE_SYSTEM_NEIGH_TABLE_NAME)
+    {
+        doStateSystemNeighTask(consumer);
+    }
+    else
     {
         SWSS_LOG_ERROR("Unknown REDIS table %s ", table_name.c_str());
     }
 }
+
+void NbrMgr::doStateSystemNeighTask(Consumer &consumer)
+{
+    SWSS_LOG_ENTER();
+
+    //Get the name of the device on which the neigh and route are
+    //going to be programmed.
+    string nbr_odev;
+    if(!getVoqInbandInterfaceName(nbr_odev))
+    {
+        //The inband interface is not available yet
+        return;
+    }
+
+    auto it = consumer.m_toSync.begin();
+    while (it != consumer.m_toSync.end())
+    {
+        KeyOpFieldsValuesTuple t = it->second;
+        string key = kfvKey(t);
+        string op = kfvOp(t);
+
+        size_t found = key.find_last_of(state_db_key_delimiter);
+        if (found == string::npos)
+        {
+            SWSS_LOG_ERROR("Failed to parse key %s", key.c_str());
+            it = consumer.m_toSync.erase(it);
+            continue;
+        }
+
+        IpAddress ip_address(key.substr(found+1));
+        if (op == SET_COMMAND)
+        {
+            MacAddress mac_address;
+            for (auto i = kfvFieldsValues(t).begin();
+                 i  != kfvFieldsValues(t).end(); i++)
+            {
+                if (fvField(*i) == "neigh")
+                    mac_address = MacAddress(fvValue(*i));
+            }
+
+            if (!isIntfStateOk(nbr_odev))
+            {
+                SWSS_LOG_DEBUG("Interface %s is not ready, skipping system neigh %s'", nbr_odev.c_str(), kfvKey(t).c_str());
+                it++;
+                continue;
+            }
+
+            if (!addKernelNeigh(nbr_odev, ip_address, mac_address))
+            {
+                SWSS_LOG_ERROR("Neigh entry add on dev %s failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+                it++;
+                continue;
+            }
+            else
+            {
+                SWSS_LOG_NOTICE("Neigh entry added on dev %s for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+
+            if (!addKernelRoute(nbr_odev, ip_address))
+            {
+                SWSS_LOG_ERROR("Route entry add on dev %s failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+                delKernelNeigh(nbr_odev, ip_address);
+                it++;
+                continue;
+            }
+            else
+            {
+                SWSS_LOG_NOTICE("Route entry added on dev %s for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+            SWSS_LOG_NOTICE("Added voq neighbor %s to kernel", kfvKey(t).c_str());
+        }
+        else if (op == DEL_COMMAND)
+        {
+            if (!delKernelRoute(ip_address))
+            {
+                SWSS_LOG_ERROR("Route entry on dev %s delete failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+            else
+            {
+                SWSS_LOG_NOTICE("Route entry on dev %s deleted for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+
+            if (!delKernelNeigh(nbr_odev, ip_address))
+            {
+                SWSS_LOG_ERROR("Neigh entry on dev %s delete failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+            else
+            {
+                SWSS_LOG_NOTICE("Neigh entry on dev %s deleted for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
+            }
+            SWSS_LOG_DEBUG("Deleted voq neighbor %s from kernel", kfvKey(t).c_str());
+        }
+
+        it = consumer.m_toSync.erase(it);
+    }
+}
+
+bool NbrMgr::getVoqInbandInterfaceName(string &ibif)
+{
+
+    vector<string> keys;
+    m_cfgVoqInbandInterfaceTable->getKeys(keys);
+
+    if (keys.empty())
+    {
+        SWSS_LOG_NOTICE("Voq Inband interface is not configured!");
+        return false;
+    }
+    //key:"alias" = inband interface name
+    vector<string> if_keys = tokenize(keys[0], config_db_key_delimiter);
+    ibif = if_keys[0];
+    return true;
+}
+
+bool NbrMgr::addKernelRoute(string odev, IpAddress ip_addr)
+{
+    string cmd, res;
+
+    SWSS_LOG_ENTER();
+
+    string ip_str = ip_addr.to_string();
+
+    if(ip_addr.isV4())
+    {
+        cmd = string("") + IP_CMD + " route add " + ip_str + "/32 dev " + odev;
+        SWSS_LOG_NOTICE("IPv4 Route Add cmd: %s",cmd.c_str());
+    }
+    else
+    {
+        cmd = string("") + IP_CMD + " -6 route add " + ip_str + "/128 dev " + odev;
+        SWSS_LOG_NOTICE("IPv6 Route Add cmd: %s",cmd.c_str());
+    }
+
+    int32_t ret = swss::exec(cmd, res);
+
+    if(ret)
+    {
+        /* Just log error and return */
+        SWSS_LOG_ERROR("Failed to add route for %s, error: %d", ip_str.c_str(), ret);
+        return false;
+    }
+
+    SWSS_LOG_INFO("Added route for %s on device %s", ip_str.c_str(), odev.c_str());
+    return true;
+}
+
+bool NbrMgr::delKernelRoute(IpAddress ip_addr)
+{
+    string cmd, res;
+
+    SWSS_LOG_ENTER();
+
+    string ip_str = ip_addr.to_string();
+
+    if(ip_addr.isV4())
+    {
+        cmd = string("") + IP_CMD + " route del " + ip_str + "/32";
+        SWSS_LOG_NOTICE("IPv4 Route Del cmd: %s",cmd.c_str());
+    }
+    else
+    {
+        cmd = string("") + IP_CMD + " -6 route del " + ip_str + "/128";
+        SWSS_LOG_NOTICE("IPv6 Route Del cmd: %s",cmd.c_str());
+    }
+
+    int32_t ret = swss::exec(cmd, res);
+
+    if(ret)
+    {
+        /* Just log error and return */
+        SWSS_LOG_ERROR("Failed to delete route for %s, error: %d", ip_str.c_str(), ret);
+        return false;
+    }
+
+    SWSS_LOG_INFO("Deleted route for %s", ip_str.c_str());
+    return true;
+}
+
+bool NbrMgr::addKernelNeigh(string odev, IpAddress ip_addr, MacAddress mac_addr)
+{
+    SWSS_LOG_ENTER();
+
+    string cmd, res;
+    string ip_str = ip_addr.to_string();
+    string mac_str = mac_addr.to_string();
+
+    if(ip_addr.isV4())
+    {
+        cmd = string("") + IP_CMD + " neigh add " + ip_str + " lladdr " + mac_str + " dev " + odev;
+        SWSS_LOG_NOTICE("IPv4 Nbr Add cmd: %s",cmd.c_str());
+    }
+    else
+    {
+        cmd = string("") + IP_CMD + " -6 neigh add " + ip_str + " lladdr " + mac_str + " dev " + odev;
+        SWSS_LOG_NOTICE("IPv6 Nbr Add cmd: %s",cmd.c_str());
+    }
+
+    int32_t ret = swss::exec(cmd, res);
+
+    if(ret)
+    {
+        /* Just log error and return */
+        SWSS_LOG_ERROR("Failed to add Nbr for %s, error: %d", ip_str.c_str(), ret);
+        return false;
+    }
+
+    SWSS_LOG_INFO("Added Nbr for %s on interface %s", ip_str.c_str(), odev.c_str());
+    return true;
+}
+
+bool NbrMgr::delKernelNeigh(string odev, IpAddress ip_addr)
+{
+    string cmd, res;
+
+    SWSS_LOG_ENTER();
+
+    string ip_str = ip_addr.to_string();
+
+    if(ip_addr.isV4())
+    {
+        cmd = string("") + IP_CMD + " neigh del " + ip_str + " dev " + odev;
+        SWSS_LOG_NOTICE("IPv4 Nbr Del cmd: %s",cmd.c_str());
+    }
+    else
+    {
+        cmd = string("") + IP_CMD + " -6 neigh del " + ip_str + " dev " + odev;
+        SWSS_LOG_NOTICE("IPv6 Nbr Del cmd: %s",cmd.c_str());
+    }
+
+    int32_t ret = swss::exec(cmd, res);
+
+    if(ret)
+    {
+        /* Just log error and return */
+        SWSS_LOG_ERROR("Failed to delete Nbr for %s, error: %d", ip_str.c_str(), ret);
+        return false;
+    }
+
+    SWSS_LOG_INFO("Deleted Nbr for %s on interface %s", ip_str.c_str(), odev.c_str());
+    return true;
+}

cfgmgr/nbrmgr.h

@@ -29,6 +29,13 @@ class NbrMgr : public Orch
     void doResolveNeighTask(Consumer &consumer);
     void doSetNeighTask(Consumer &consumer);
     void doTask(Consumer &consumer);
+    void doStateSystemNeighTask(Consumer &consumer);
+    bool getVoqInbandInterfaceName(string &nbr_odev);
+    bool addKernelRoute(string odev, IpAddress ip_addr);
+    bool delKernelRoute(IpAddress ip_addr);
+    bool addKernelNeigh(string odev, IpAddress ip_addr, MacAddress mac_addr);
+    bool delKernelNeigh(string odev, IpAddress ip_addr);
+    unique_ptr<Table> m_cfgVoqInbandInterfaceTable;
 
     Table m_statePortTable, m_stateLagTable, m_stateVlanTable, m_stateIntfTable, m_stateNeighRestoreTable;
     struct nl_sock *m_nl_sock;