Merge pull request zhenggen-xu#7 from cisco-n3000-LI/202012_rebase_10…

…Apr22

202012 rebase to 10Apr22

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -14,14 +14,46 @@ schedules:
     - 202012
   always: true
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 pool: sonicbld
 
+parameters:
+  - name: jobFilters
+    type: object
+    default:
+    - vs
+    - broadcom
+    - barefoot
+    - centec
+    - generic
+    - innovium
+    - mellanox
+    - nephos
+    - centec-arm64
+    - marvell-armhf
+
 stages:
 - stage: Build
+  variables:
+    - name: CACHE_MODE
+      value: none
+    - name: VERSION_CONTROL_OPTIONS
+      value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
-      buildSlave: y
+      jobFilters: ${{ parameters.jobFilters }}
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      preSteps:
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions
@@ -41,18 +73,24 @@ stages:
     - download: current
       patterns: '**/versions-*'
     - script: |
+        # Cleanup the old versions
+        rm -rf files/build/versions
+
         mkdir -p target
         default_platform=broadcom
         artifacts=$(find $(Pipeline.Workspace) -maxdepth 1 -type d -name 'sonic-buildimage.*' | grep -v "sonic-buildimage.${default_platform}")
         echo "artifacts$artifacts"
-        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
+        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/target/versions target/
         make freeze FREEZE_VERSION_OPTIONS=-r
         find files/build/versions 
-        for artifact in $artifacts
+        ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
+        for artifact in $ordered_artifacts
         do
           rm -rf target/versions
-          cp -r $artifact/versions target/
-          make freeze FREEZE_VERSION_OPTIONS="-a -d"
+          cp -r $artifact/target/versions target/
+          OPTIONS="-a -d"
+          [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
+          make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"
         done
         git diff files/build/versions
       displayName: 'Freeze Versions'

.azure-pipelines/azure-pipelines-build.yml

@@ -1,72 +1,133 @@
+# The azure pipeline template for Official build, and upgrade version build
+
 parameters:
 - name: 'jobFilters'
   type: object
   default: ''
 - name: 'buildOptions'
   type: string
   default: 'SONIC_CONFIG_BUILD_JOBS=1'
-- name: 'buildSlave'
-  type: string
-  default: 'n'
+- name: 'preSteps'
+  type: stepList
+  default: []
 - name: 'postSteps'
   type: stepList
   default: []
+- name: 'jobGroups'
+  type: object
+  default: ''
 
 jobs:
-- template: azure-pipelines-job-groups.yml
+- template: azure-pipelines-image-template.yml
   parameters:
     jobFilters: ${{ parameters.jobFilters }}
-    preSteps:
-      - script: |
-          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-          if [ ! -z "$containers" ]; then
-            docker container kill $containers || true
-            sleep 5
-          fi
-          if [ "${{ parameters.buildSlave }}" == "y" ]; then
-            images=$(docker images 'sonic-slave-*' -a -q)
-            [ ! -z "$images" ] && docker rmi -f $images
+    preSteps: ${{ parameters.preSteps }}
+    postSteps: ${{ parameters.postSteps }}
+    jobVariables:
+      PLATFORM_AZP: $(GROUP_NAME)
+      PLATFORM_ARCH: amd64
+      BUILD_NUMBER: $(Build.BuildId)
+      BUILD_OPTIONS: ${{ parameters.buildOptions }}
+      DOCKER_DATA_ROOT_FOR_MULTIARCH: /data/march/docker
+      dbg_image: no
+      swi_image: no
+      raw_image: no
+      docker_syncd_rpc_image: no
+      syncd_rpc_image: no
+      platform_rpc: no
+    ${{ if ne(parameters.jobGroups, '') }}:
+      jobGroups: ${{ parameters.jobGroups }}
+    ${{ if eq(parameters.jobGroups, '') }}:
+      jobGroups:
+        - name: vs
+          variables:
+            dbg_image: yes
+
+        - name: barefoot
+          variables:
+            docker_syncd_rpc_image: yes
+            platform_rpc: bfn
+            swi_image: yes
+
+        - name: broadcom
+          variables:
+            dbg_image: yes
+            swi_image: yes
+            raw_image: yes
+            docker_syncd_rpc_image: yes
+            platform_rpc: brcm
+
+        - name: centec
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: yes
+            platform_rpc: centec
+
+        - name: centec-arm64
+          pool: sonicbld-arm64
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: arm64
+            BUILD_OPTIONS: ${{ parameters.buildOptions }} SONIC_VERSION_CONTROL_COMPONENTS=
+        - name: generic
+          variables:
+            dbg_image: yes
+
+        - name: innovium
+          variables:
+            dbg_image: yes
+
+        - name: marvell-armhf
+          pool: sonicbld-armhf
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: armhf
+            BUILD_OPTIONS: ${{ parameters.buildOptions }} SONIC_VERSION_CONTROL_COMPONENTS=
+        - name: mellanox
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: yes
+            syncd_rpc_image: yes
+            platform_rpc: mlnx
+
+        - name: nephos
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: no
+            platform_rpc: nephos
+
+    buildSteps:
+      - bash: |
+          set -ex
+          if [ $(GROUP_NAME) == vs ]; then
+            if [ $(dbg_image) == yes ]; then
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+            fi
+            make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            if [ $(Build.Reason) != 'PullRequest' ];then
+              gzip -kd target/sonic-vs.img.gz
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              rm target/sonic-vs.img
+            fi
+          else
+            if [ $(dbg_image) == yes ]; then
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+            fi
+            if [ $(swi_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
+            fi
+            if [ $(raw_image) == yes ]; then
+              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
+            fi
+            if [ $(docker_syncd_rpc_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+            fi
+            if [ $(syncd_rpc_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-mellanox.bin target/sonic-$(GROUP_NAME)-rpc.bin
+            fi
+            make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
-          sudo rm -rf $(ls -A1)
-        displayName: 'Init'
-      - checkout: self
-        submodules: recursive
-        displayName: 'Checkout code'
-      - script: |
-          make ${{ parameters.buildOptions }} PLATFORM=$GROUP_NAME configure
-        displayName: 'Make configure'
-    postSteps:
-      - ${{ parameters.postSteps }}
-      - publish: $(System.DefaultWorkingDirectory)/target
-        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
-        displayName: "Archive sonic image"
-    jobGroups:
-      - name: vs
-        script: |
-          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
-          make ${{ parameters.buildOptions }} target/sonic-vs.img.gz
-      - name: broadcom
-        script: |
-          make ${{ parameters.buildOptions }} target/sonic-broadcom.bin target/sonic-aboot-broadcom.swi
-      - name: barefoot
-        script: |
-          make ${{ parameters.buildOptions }} target/sonic-barefoot.bin target/sonic-aboot-barefoot.swi
-      - name: centec
-        script: |
-          make ${{ parameters.buildOptions }} INSTALL_DEBUG_TOOLS=y target/sonic-centec.bin
-          mv target/sonic-centec.bin target/sonic-centec-dbg.bin
-          make ${{ parameters.buildOptions }} target/sonic-centec.bin
-          make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y target/docker-syncd-centec-rpc.gz
-      - name: innovium
-        script: |
-          make ${{ parameters.buildOptions }} SONIC_CONFIG_BUILD_JOBS=1 target/sonic-innovium.bin
-      - name: mellanox
-        script: |
-          make ${{ parameters.buildOptions }} target/sonic-mellanox.bin
-      - name: mellanox
-        extName: _rpc
-        script: |
-          make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y all
-      - name: nephos
-        script: |
-          make ${{ parameters.buildOptions }} target/sonic-nephos.bin
+        displayName: "Build sonic image"

.azure-pipelines/azure-pipelines-image-template.yml

@@ -0,0 +1,59 @@
+# The azure pipeline template for PR build, Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'buildSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: jobGroups
+  type: object
+  default: []
+- name: jobVariables
+  type: object
+  default: []
+jobs:
+- template: azure-pipelines-job-groups.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    jobVariables: ${{ parameters.jobVariables }}
+    preSteps:
+      - template: cleanup.yml
+      - ${{ parameters.preSteps }}
+      - script: |
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
+            CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
+            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
+            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+          fi
+        displayName: "Set cache options"
+      - checkout: self
+        submodules: recursive
+        condition: eq(variables.SKIP_CHECKOUT, '') 
+        displayName: 'Checkout code'
+      - script: |
+          BRANCH_NAME=$(Build.SourceBranchName)
+          [ -n "$SYSTEM_PULLREQUEST_PULLREQUESTID" ] && BRANCH_NAME="$SYSTEM_PULLREQUEST_TARGETBRANCH-$SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"
+          git checkout -b $BRANCH_NAME
+          sudo modprobe overlay
+          sudo apt-get install -y acl
+          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
+          ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
+        displayName: 'Make configure'
+    postSteps:
+      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+        displayName: Copy Artifacts
+      - publish:  $(Build.ArtifactStagingDirectory)
+        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+        displayName: "Archive sonic image"
+      - ${{ parameters.postSteps }}
+      - template: cleanup.yml
+    jobGroups: ${{ parameters.jobGroups }}
+    buildSteps: ${{ parameters.buildSteps }}

.azure-pipelines/azure-pipelines-job-groups.yml

@@ -2,6 +2,9 @@ parameters:
 - name: 'preSteps'
   type: stepList
   default: []
+- name: 'buildSteps'
+  type: stepList
+  default: []
 - name: "postSteps"
   type: stepList
   default: []
@@ -16,14 +19,14 @@ parameters:
   default: ''
 - name: 'timeoutInMinutes'
   type: 'number'
-  default: 1440
+  default: 720
 - name: 'jobFilters'
   type: object
   default: ''
 
 jobs:
 - ${{ each jobGroup in parameters.jobGroups }}:
-  - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name)) }}:
+  - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name), endswith(variables['Build.DefinitionName'], format('.{0}{1}', jobGroup.name, jobGroup.extName))) }}:
     - job: ${{ replace(format('{0}{1}', jobGroup.name, jobGroup.extName), '-', '_') }}
       ${{ each pair in jobGroup }}:
         ${{ if not(in(pair.key, 'job', 'name', 'extName', 'variables', 'steps', 'script', 'scriptEnv')) }}:
@@ -50,4 +53,5 @@ jobs:
           displayName: 'JobScript'
       - ${{ if ne(jobGroup.steps, '') }}:
         - ${{ jobGroup.steps }}
+      - ${{ parameters.buildSteps }}
       - ${{ parameters.postSteps }}

.azure-pipelines/build-commonlib.yml

@@ -0,0 +1,19 @@
+pr: none
+trigger: none
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+jobs:
+- template: .azure-pipelines/template-commonlib.yml@buildimage

.azure-pipelines/build-template.yml

@@ -114,3 +114,12 @@ jobs:
   - publish: $(System.DefaultWorkingDirectory)/
     artifact: sonic-buildimage.${{ parameters.platform }}
     displayName: "Archive sonic image"
+  - script: |
+      set -x
+      find target -name "*.log" | xargs -I{} cp {} $(Build.ArtifactStagingDirectory)/
+    condition: failed()
+    displayName: "Copy build logs"
+  - publish: $(Build.ArtifactStagingDirectory)/
+    artifact: sonic-buildimage.${{ parameters.platform}}.logs@$(System.JobAttempt)
+    displayName: "Archive build logs"
+    condition: failed()