Merge branch 'sonic-net:master' into master

xumia · Apr 9, 2024 · aae34cf · aae34cf
2 parents bd03f69 + 56ba6de
commit aae34cf
Show file tree

Hide file tree

Showing 344 changed files with 38,789 additions and 5,191 deletions.
diff --git a/.artifactignore b/.artifactignore
@@ -1,2 +1,5 @@
 **/*
 !*.deb
+!coverage.info
+!coverage.xml
+!build.info
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -14,15 +14,55 @@ parameters:
 - name: swss_artifact_name
   type: string
 
-- name: sairedis_artifact_name
+- name: swss_common_artifact_name
   type: string
 
-- name: swss_common_artifact_name
+- name: swss_common_branch
   type: string
+  default: '$(BUILD_BRANCH)'
 
 - name: artifact_name
   type: string
 
+- name: buildimage_artifact_name
+  type: string
+  default: 'sonic-buildimage.vs'
+
+- name: buildimage_artifact_project
+  type: string
+  default: 'build'
+
+- name: buildimage_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-buildimage.official.vs'
+
+- name: buildimage_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_name
+  type: string
+
+- name: sairedis_artifact_project
+  type: string
+  default: 'build'
+
+- name: sairedis_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-sairedis'
+
+- name: sairedis_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_pattern
+  type: string
+  default: '**'
+
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -39,19 +79,42 @@ jobs:
       pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      runBranch: 'refs/heads/${{ parameters.swss_common_branch }}'
       path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-sairedis
+      project: ${{ parameters.sairedis_artifact_project }}
+      pipeline: ${{ parameters.sairedis_artifact_pipeline }}
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/${{ parameters.sairedis_artifact_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/sairedis
+      patterns: |
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/syncd-vs_*.deb
+    displayName: "Download sonic sairedis deb packages"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-dash-api
+      ${{ if eq(parameters.arch, 'amd64') }}:
+        artifact: sonic-dash-api
+      ${{ else }}:
+        artifact: sonic-dash-api.${{ parameters.arch }}
+      runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
-    displayName: "Download sonic sairedis deb packages"
+      patterns: |
+        libdashapi*.deb
+    displayName: "Download dash api"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.swss_artifact_name }}
@@ -60,33 +123,44 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.official.vs
-      artifact: sonic-buildimage.vs
+      project: ${{ parameters.buildimage_artifact_project }}
+      pipeline: ${{ parameters.buildimage_artifact_pipeline }}
+      artifact: ${{ parameters.buildimage_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      runBranch: 'refs/heads/${{ parameters.buildimage_artifact_branch }}'
       path: $(Build.ArtifactStagingDirectory)/download
-      patterns: '**/target/docker-sonic-vs.gz'
-    displayName: "Download sonic-buildimage docker-sonic-vs"
+      patterns: '**/target/${{ parameters.artifact_name }}.gz'
+    displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
   - script: |
       set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
+      find $(Build.ArtifactStagingDirectory)/download/sairedis -name '*.deb' -exec cp "{}" .azure-pipelines/docker-sonic-vs/debs \;
       cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
+      if [ -f $(Build.ArtifactStagingDirectory)/download/coverage.info ]; then
+        cp -v $(Build.ArtifactStagingDirectory)/download/coverage.info $(Build.ArtifactStagingDirectory)/
+        cp -v $(Build.ArtifactStagingDirectory)/download/coverage.xml $(Build.ArtifactStagingDirectory)/
+      fi
 
       pushd .azure-pipelines
 
-      docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
+      build_dir=$(grep BUILD_DIR $(Build.ArtifactStagingDirectory)/download/build.info | cut -d= -f2)
+      build_args="--build-arg build_dir=$build_dir"
+      if [ '${{ parameters.asan }}' == True ]; then
+        build_args="$build_args --build-arg need_dbg=y"
+      fi
+
+      docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs
 
       popd
 
-      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
+      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
       rm -rf $(Build.ArtifactStagingDirectory)/download
-    displayName: "Build docker-sonic-vs"
+    displayName: "Build ${{ parameters.artifact_name }}"
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive sonic docker vs image"
diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -26,9 +26,29 @@ parameters:
 - name: sairedis_artifact_name
   type: string
 
+- name: sairedis_artifact_project
+  type: string
+  default: 'build'
+
+- name: sairedis_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-sairedis'
+
+- name: sairedis_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: sairedis_artifact_pattern
+  type: string
+  default: '**'
+
 - name: swss_common_artifact_name
   type: string
 
+- name: swss_common_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
 - name: artifact_name
   type: string
 
@@ -43,6 +63,22 @@ parameters:
 - name: common_lib_artifact_name
   type: string
 
+- name: common_lib_artifact_project
+  type: string
+  default: 'build'
+
+- name: common_lib_artifact_pipeline
+  type: string
+  default: 'Azure.sonic-buildimage.common_libs'
+
+- name: common_lib_artifact_branch
+  type: string
+  default: '$(BUILD_BRANCH)'
+
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -62,15 +98,14 @@ jobs:
     clean: true
     submodules: true
   - script: |
-      sudo apt-get install -y libhiredis0.14 libhiredis-dev
-      sudo apt-get install -y libzmq5 libzmq3-dev
-      sudo apt-get install -qq -y \
-          libhiredis-dev \
-          swig3.0
-      sudo apt-get install -y libdbus-1-3
-      sudo apt-get install -y libteam-dev \
-          libteam5 \
-          libteamdctl0
+      sudo apt-get update
+      sudo apt-get install -y \
+        libhiredis-dev \
+        libzmq3-dev \
+        swig4.0 \
+        libdbus-1-dev \
+        libteam-dev
+      sudo pip3 install lcov_cobertura
     displayName: "Install dependencies"
   - task: DownloadPipelineArtifact@2
     inputs:
@@ -79,54 +114,76 @@ jobs:
       pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
+      runBranch: 'refs/heads/${{ parameters.swss_common_branch }}'
+      allowPartiallySucceededBuilds: true
+      path: $(Build.ArtifactStagingDirectory)/download/swsscommon
       patterns: |
         libswsscommon_1.0.0_${{ parameters.arch }}.deb
         libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-sairedis
+      project: ${{ parameters.sairedis_artifact_project }}
+      pipeline: ${{ parameters.sairedis_artifact_pipeline }}
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
+      runBranch: 'refs/heads/${{ parameters.sairedis_artifact_branch }}'
+      allowPartiallySucceededBuilds: true
+      path: $(Build.ArtifactStagingDirectory)/download/sairedis
       patterns: |
-        libsaivs_*.deb
-        libsaivs-dev_*.deb
-        libsairedis_*.deb
-        libsairedis-dev_*.deb
-        libsaimetadata_*.deb
-        libsaimetadata-dev_*.deb
-        syncd-vs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaivs-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsairedis-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/libsaimetadata-dev_*.deb
+        ${{ parameters.sairedis_artifact_pattern }}/syncd-vs_*.deb
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
-      project: build
-      pipeline: Azure.sonic-buildimage.common_libs
+      project: ${{ parameters.common_lib_artifact_project }}
+      pipeline: ${{ parameters.common_lib_artifact_pipeline }}
       artifact: ${{ parameters.common_lib_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/$(BUILD_BRANCH)'
-      path: $(Build.ArtifactStagingDirectory)/download
+      runBranch: 'refs/heads/${{ parameters.common_lib_artifact_branch }}'
+      path: $(Build.ArtifactStagingDirectory)/download/common
       patterns: |
-        target/debs/buster/libnl-3-200_*.deb
-        target/debs/buster/libnl-3-dev_*.deb
-        target/debs/buster/libnl-genl-3-200_*.deb
-        target/debs/buster/libnl-genl-3-dev_*.deb
-        target/debs/buster/libnl-route-3-200_*.deb
-        target/debs/buster/libnl-route-3-dev_*.deb
-        target/debs/buster/libnl-nf-3-200_*.deb
-        target/debs/buster/libnl-nf-3-dev_*.deb
+        target/debs/bullseye/libnl-3-200_*.deb
+        target/debs/bullseye/libnl-3-dev_*.deb
+        target/debs/bullseye/libnl-genl-3-200_*.deb
+        target/debs/bullseye/libnl-genl-3-dev_*.deb
+        target/debs/bullseye/libnl-route-3-200_*.deb
+        target/debs/bullseye/libnl-route-3-dev_*.deb
+        target/debs/bullseye/libnl-nf-3-200_*.deb
+        target/debs/bullseye/libnl-nf-3-dev_*.deb
+        target/debs/bullseye/libyang_*.deb
+        target/debs/bullseye/libprotobuf*.deb
+        target/debs/bullseye/libprotoc*.deb
+        target/debs/bullseye/protobuf-compiler*.deb
     displayName: "Download common libs"
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: specific
+      project: build
+      pipeline: sonic-net.sonic-dash-api
+      ${{ if eq(parameters.arch, 'amd64') }}:
+        artifact: sonic-dash-api
+      ${{ else }}:
+        artifact: sonic-dash-api.${{ parameters.arch }}
+      runVersion: 'latestFromBranch'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download/common
+      patterns: |
+        libdashapi*.deb
+    displayName: "Download dash api"
   - script: |
       set -ex
       cd download
-      sudo dpkg -i $(find target/debs/buster -type f)
-      sudo dpkg -i $(ls *.deb)
+      sudo dpkg -i $(find common -type f -name '*.deb')
+      sudo dpkg -i $(find swsscommon -type f -name '*.deb')
+      sudo dpkg -i $(find sairedis -type f -name '*.deb')
       cd ..
       rm -rf download
     workingDirectory: $(Build.ArtifactStagingDirectory)
@@ -137,6 +194,10 @@ jobs:
       cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/
       if [ '${{ parameters.archive_gcov }}' == True ]; then
          export ENABLE_GCOV=y
+         echo BUILD_DIR=$(pwd) > build.info
+      fi
+      if [ '${{ parameters.asan }}' == True ]; then
+         export ENABLE_ASAN=y
       fi
       ./autogen.sh
       dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .