Merge branch 'master' into sag-dev

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -45,6 +45,7 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
+      allowPartiallySucceededBuilds: true
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -55,6 +56,7 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
+      allowPartiallySucceededBuilds: true
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:

.azure-pipelines/build-template.yml

@@ -66,15 +66,13 @@ jobs:
     clean: true
     submodules: true
   - script: |
-      sudo apt-get install -y libhiredis0.14 libhiredis-dev
-      sudo apt-get install -y libzmq5 libzmq3-dev
-      sudo apt-get install -qq -y \
-          libhiredis-dev \
-          swig3.0
-      sudo apt-get install -y libdbus-1-3
-      sudo apt-get install -y libteam-dev \
-          libteam5 \
-          libteamdctl0
+      sudo apt-get update
+      sudo apt-get install -y \
+        libhiredis-dev \
+        libzmq3-dev \
+        swig4.0 \
+        libdbus-1-dev \
+        libteam-dev
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -85,6 +83,7 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
+      allowPartiallySucceededBuilds: true
       patterns: |
         libswsscommon_1.0.0_${{ parameters.arch }}.deb
         libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
@@ -98,6 +97,7 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
+      allowPartiallySucceededBuilds: true
       patterns: |
         libsaivs_*.deb
         libsaivs-dev_*.deb
@@ -117,20 +117,20 @@ jobs:
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
       patterns: |
-        target/debs/buster/libnl-3-200_*.deb
-        target/debs/buster/libnl-3-dev_*.deb
-        target/debs/buster/libnl-genl-3-200_*.deb
-        target/debs/buster/libnl-genl-3-dev_*.deb
-        target/debs/buster/libnl-route-3-200_*.deb
-        target/debs/buster/libnl-route-3-dev_*.deb
-        target/debs/buster/libnl-nf-3-200_*.deb
-        target/debs/buster/libnl-nf-3-dev_*.deb
-        target/debs/buster/libyang_*.deb
+        target/debs/bullseye/libnl-3-200_*.deb
+        target/debs/bullseye/libnl-3-dev_*.deb
+        target/debs/bullseye/libnl-genl-3-200_*.deb
+        target/debs/bullseye/libnl-genl-3-dev_*.deb
+        target/debs/bullseye/libnl-route-3-200_*.deb
+        target/debs/bullseye/libnl-route-3-dev_*.deb
+        target/debs/bullseye/libnl-nf-3-200_*.deb
+        target/debs/bullseye/libnl-nf-3-dev_*.deb
+        target/debs/bullseye/libyang_*.deb
     displayName: "Download common libs"
   - script: |
       set -ex
       cd download
-      sudo dpkg -i $(find target/debs/buster -type f)
+      sudo dpkg -i $(find target/debs/bullseye -type f)
       sudo dpkg -i $(ls *.deb)
       cd ..
       rm -rf download

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -3,22 +3,17 @@ FROM docker-sonic-vs
 ARG docker_container_name
 ARG need_dbg
 
-ADD ["debs", "/debs"]
+COPY ["debs", "/debs"]
 
-RUN dpkg --purge python-swsscommon python3-swsscommon swss libsairedis sonic-db-cli libswsscommon libsaimetadata libsaivs syncd-vs
+RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb \
+            /debs/python3-swsscommon_1.0.0_amd64.deb \
+            /debs/sonic-db-cli_1.0.0_amd64.deb \
+            /debs/libsaimetadata_1.0.0_amd64.deb \
+            /debs/libsairedis_1.0.0_amd64.deb \
+            /debs/libsaivs_1.0.0_amd64.deb \
+            /debs/syncd-vs_1.0.0_amd64.deb \
+            /debs/swss_1.0.0_amd64.deb
 
-RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/python-swsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/python3-swsscommon_1.0.0_amd64.deb
-RUN dpkg -i /debs/sonic-db-cli_1.0.0_amd64.deb
-
-RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb
-RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb
-RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb
-RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
-
-RUN dpkg --purge swss
-RUN dpkg -i /debs/swss_1.0.0_amd64.deb
 RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi
 
 RUN apt-get update

.azure-pipelines/gcov.yml

@@ -59,9 +59,9 @@ jobs:
         set -ex
         # Install .NET CORE
         curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-        sudo apt-add-repository https://packages.microsoft.com/debian/10/prod
+        sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
         sudo apt-get update
-        sudo apt-get install -y dotnet-sdk-5.0
+        sudo apt-get install -y dotnet-sdk-7.0
     displayName: "Install .NET CORE"
   - script: |
       sudo apt-get install -y lcov

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -63,7 +63,7 @@ jobs:
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
       sudo apt-get install -y python3-pip
-      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0
+      sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0
     displayName: "Install dependencies"
 
   - script: |

.gitignore

@@ -76,6 +76,8 @@ teamsyncd/teamsyncd
 tests/tests
 tests/mock_tests/tests_response_publisher
 tests/mock_tests/tests_fpmsyncd
+tests/mock_tests/tests_intfmgrd
+tests/mock_tests/tests_portsyncd
 
 
 # Test Files #
@@ -87,5 +89,7 @@ tests/mock_tests/tests.trs
 tests/test-suite.log
 tests/tests.log
 tests/tests.trs
+tests/mock_tests/**/*log
+tests/mock_tests/**/*trs
 orchagent/p4orch/tests/**/*gcda
 orchagent/p4orch/tests/**/*gcno

azure-pipelines.yml

@@ -15,6 +15,7 @@ trigger:
     - master
     - 202???
     - 201???
+    - dash
 
 # this part need to be set in UI
 schedules:
@@ -41,7 +42,7 @@ stages:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: amd64
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-bullseye
       common_lib_artifact_name: common-lib
       swss_common_artifact_name: sonic-swss-common
       sairedis_artifact_name: sonic-sairedis
@@ -55,7 +56,7 @@ stages:
   - template: .azure-pipelines/build-template.yml
     parameters:
       arch: amd64
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-bullseye
       common_lib_artifact_name: common-lib
       swss_common_artifact_name: sonic-swss-common
       sairedis_artifact_name: sonic-sairedis
@@ -71,7 +72,7 @@ stages:
       arch: armhf
       timeout: 240
       pool: sonicbld-armhf
-      sonic_slave: sonic-slave-buster-armhf
+      sonic_slave: sonic-slave-bullseye-armhf
       common_lib_artifact_name: common-lib.armhf
       swss_common_artifact_name: sonic-swss-common.armhf
       sairedis_artifact_name: sonic-sairedis.armhf
@@ -83,7 +84,7 @@ stages:
       arch: arm64
       timeout: 240
       pool: sonicbld-arm64
-      sonic_slave: sonic-slave-buster-arm64
+      sonic_slave: sonic-slave-bullseye-arm64
       common_lib_artifact_name: common-lib.arm64
       swss_common_artifact_name: sonic-swss-common.arm64
       sairedis_artifact_name: sonic-sairedis.arm64
@@ -121,7 +122,7 @@ stages:
     parameters:
       log_artifact_name: log
       gcov_artifact_name: sonic-gcov
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-bullseye
       archive_gcov: true
 
 - stage: TestAsan
@@ -132,7 +133,7 @@ stages:
     parameters:
       log_artifact_name: log-asan
       gcov_artifact_name: sonic-gcov
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-bullseye
       docker_sonic_vs_name: docker-sonic-vs-asan
       asan: true
 
@@ -143,7 +144,7 @@ stages:
   - template: .azure-pipelines/gcov.yml
     parameters:
       arch: amd64
-      sonic_slave: sonic-slave-buster
+      sonic_slave: sonic-slave-bullseye
       swss_common_artifact_name: sonic-swss-common
       sairedis_artifact_name: sonic-sairedis
       swss_artifact_name: sonic-swss

cfgmgr/buffer_check_headroom_mellanox.lua

@@ -5,7 +5,7 @@
 
 local port = KEYS[1]
 local input_profile_name = ARGV[1]
-local input_profile_size = ARGV[2]
+local input_profile_size = tonumber(ARGV[2])
 local new_pg = ARGV[3]
 
 local function is_port_with_8lanes(lanes)
@@ -60,7 +60,8 @@ if is_port_with_8lanes(lanes) then
     pipeline_latency = pipeline_latency * 2 - 1
     egress_mirror_size = egress_mirror_size * 2
 end
-accumulative_size = accumulative_size + 2 * pipeline_latency * 1024 + egress_mirror_size
+local lossy_pg_size = pipeline_latency * 1024
+accumulative_size = accumulative_size + lossy_pg_size + egress_mirror_size
 
 -- Fetch all keys in BUFFER_PG according to the port
 redis.call('SELECT', appl_db)
@@ -81,41 +82,48 @@ local function get_number_of_pgs(keyname)
     return size
 end
 
-local no_input_pg = true
-if new_pg ~= nil then
-    if get_number_of_pgs(new_pg) ~= 0 then
-        no_input_pg = false
-        new_pg = 'BUFFER_PG_TABLE:' .. new_pg
-    end
+-- Fetch all the PGs in APPL_DB, and store them into a hash table
+local pg_keys = redis.call('KEYS', 'BUFFER_PG_TABLE:' .. port .. ':*')
+local all_pgs = {}
+for i = 1, #pg_keys do
+    local profile = redis.call('HGET', pg_keys[i], 'profile')
+    all_pgs[pg_keys[i]] = profile
+end
+
+-- Fetch all the pending PGs, and store them into the hash table
+-- Overwrite any existing entries
+local pending_pg_keys = redis.call('KEYS', '_BUFFER_PG_TABLE:' .. port .. ':*')
+for i = 1, #pending_pg_keys do
+    local profile = redis.call('HGET', pending_pg_keys[i], 'profile')
+    -- Remove the leading underscore when storing it into the hash table
+    all_pgs[string.sub(pending_pg_keys[i], 2, -1)] = profile
+    table.insert(debuginfo, 'debug:pending entry: ' .. pending_pg_keys[i] .. ':' .. profile)
+end
+
+if new_pg ~= nil and get_number_of_pgs(new_pg) ~= 0 then
+    all_pgs['BUFFER_PG_TABLE:' .. new_pg] = input_profile_name
 end
 
--- Fetch all the PGs, accumulate the sizes
+-- Handle all the PGs, accumulate the sizes
 -- Assume there is only one lossless profile configured among all PGs on each port
 table.insert(debuginfo, 'debug:other overhead:' .. accumulative_size)
-local pg_keys = redis.call('KEYS', 'BUFFER_PG_TABLE:' .. port .. ':*')
-for i = 1, #pg_keys do
-    local profile = redis.call('HGET', pg_keys[i], 'profile')
+for pg_key, profile in pairs(all_pgs) do
     local current_profile_size
-    if profile ~= 'ingress_lossy_profile' and (no_input_pg or new_pg ~= pg_keys[i]) then
-        if profile ~= input_profile_name and not no_input_pg then
-            local referenced_profile = redis.call('HGETALL', 'BUFFER_PROFILE_TABLE:' .. profile)
-            for j = 1, #referenced_profile, 2 do
-                if referenced_profile[j] == 'size' then
-                    current_profile_size = tonumber(referenced_profile[j+1])
-                end
-            end
-        else
-            current_profile_size = input_profile_size
-            profile = input_profile_name
+    if profile ~= input_profile_name then
+        local referenced_profile_size = redis.call('HGET', 'BUFFER_PROFILE_TABLE:' .. profile, 'size')
+        if not referenced_profile_size then
+            referenced_profile_size = redis.call('HGET', '_BUFFER_PROFILE_TABLE:' .. profile, 'size')
+            table.insert(debuginfo, 'debug:pending profile: ' .. profile)
         end
-        accumulative_size = accumulative_size + current_profile_size * get_number_of_pgs(pg_keys[i])
-        table.insert(debuginfo, 'debug:' .. pg_keys[i] .. ':' .. profile .. ':' .. current_profile_size .. ':' .. get_number_of_pgs(pg_keys[i]) .. ':accu:' .. accumulative_size)
+        current_profile_size = tonumber(referenced_profile_size)
+    else
+        current_profile_size = input_profile_size
     end
-end
-
-if not no_input_pg then
-    accumulative_size = accumulative_size + input_profile_size * get_number_of_pgs(new_pg)
-    table.insert(debuginfo, 'debug:' .. new_pg .. '*:' .. input_profile_name .. ':' .. input_profile_size .. ':' .. get_number_of_pgs(new_pg) .. ':accu:' .. accumulative_size)
+    if current_profile_size == 0 then
+        current_profile_size = lossy_pg_size
+    end
+    accumulative_size = accumulative_size + current_profile_size * get_number_of_pgs(pg_key)
+    table.insert(debuginfo, 'debug:' .. pg_key .. ':' .. profile .. ':' .. current_profile_size .. ':' .. get_number_of_pgs(pg_key) .. ':accu:' .. accumulative_size)
 end
 
 if max_headroom_size > accumulative_size then

cfgmgr/buffer_pool_mellanox.lua

@@ -324,7 +324,7 @@ for name in pairs(profiles) do
                size = size + lossypg_reserved
             end
             if size ~= 0 then
-                if shp_enabled and shp_size == 0 then
+                if shp_size == 0 then
                     local xon = tonumber(redis.call('HGET', name, 'xon'))
                     local xoff = tonumber(redis.call('HGET', name, 'xoff'))
                     if xon ~= nil and xoff ~= nil and xon + xoff > size then
@@ -346,6 +346,12 @@ accumulative_occupied_buffer = accumulative_occupied_buffer + lossypg_extra_for_
 
 -- Accumulate sizes for private headrooms
 local accumulative_private_headroom = 0
+local force_enable_shp = false
+if accumulative_xoff > 0 and shp_enabled ~= true then
+    force_enable_shp = true
+    shp_size = 655360
+    shp_enabled = true
+end
 if shp_enabled then
     accumulative_private_headroom = lossless_port_count * private_headroom
     accumulative_occupied_buffer = accumulative_occupied_buffer + accumulative_private_headroom
@@ -391,6 +397,9 @@ end
 
 if shp_enabled and shp_size == 0 then
     shp_size = math.ceil(accumulative_xoff / over_subscribe_ratio)
+    if shp_size == 0 then
+        shp_size = 655360
+    end
 end
 
 local pool_size
@@ -432,6 +441,7 @@ table.insert(result, "debug:mgmt_pool:" .. mgmt_pool_size)
 if shp_enabled then
     table.insert(result, "debug:accumulative_private_headroom:" .. accumulative_private_headroom)
     table.insert(result, "debug:accumulative xoff:" .. accumulative_xoff)
+    table.insert(result, "debug:force enabled shp:" .. tostring(force_enable_shp))
 end
 table.insert(result, "debug:accumulative_mgmt_pg:" .. accumulative_management_pg)
 table.insert(result, "debug:egress_mirror:" .. accumulative_egress_mirror_overhead)

cfgmgr/buffermgrd.cpp

@@ -189,6 +189,8 @@ int main(int argc, char **argv)
             WarmStart::initialize("buffermgrd", "swss");
             WarmStart::checkWarmStart("buffermgrd", "swss");
 
+            DBConnector applStateDb("APPL_STATE_DB", 0);
+
             vector<TableConnector> buffer_table_connectors = {
                 TableConnector(&cfgDb, CFG_PORT_TABLE_NAME),
                 TableConnector(&cfgDb, CFG_PORT_CABLE_LEN_TABLE_NAME),
@@ -202,7 +204,7 @@ int main(int argc, char **argv)
                 TableConnector(&stateDb, STATE_BUFFER_MAXIMUM_VALUE_TABLE),
                 TableConnector(&stateDb, STATE_PORT_TABLE_NAME)
             };
-            cfgOrchList.emplace_back(new BufferMgrDynamic(&cfgDb, &stateDb, &applDb, buffer_table_connectors, peripherial_table_ptr, zero_profiles_ptr));
+            cfgOrchList.emplace_back(new BufferMgrDynamic(&cfgDb, &stateDb, &applDb, &applStateDb, buffer_table_connectors, peripherial_table_ptr, zero_profiles_ptr));
         }
         else if (!pg_lookup_file.empty())
         {