Upgrade new DellEMC platforms to Bullseye

.azure-pipelines/azure-pipelines-build.yml

@@ -64,7 +64,7 @@ jobs:
             platform_rpc: centec
 
         - name: centec-arm64
-          pool: sonicbld_8c
+          pool: sonicbld-arm64
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: arm64
@@ -78,7 +78,7 @@ jobs:
             dbg_image: yes
 
         - name: marvell-armhf
-          pool: sonicbld_8c
+          pool: sonicbld-armhf
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: armhf
@@ -104,6 +104,11 @@ jobs:
               make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
             fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            if [ $(Build.Reason) != 'PullRequest' ];then
+              gzip -kd target/sonic-vs.img.gz
+              SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
+              rm target/sonic-vs.img
+            fi
           else
             if [ $(dbg_image) == yes ]; then
               make  $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-$(GROUP_NAME).bin && \

.azure-pipelines/azure-pipelines-image-template.yml

@@ -26,7 +26,7 @@ jobs:
     jobVariables: ${{ parameters.jobVariables }}
     preSteps:
       - template: cleanup.yml
-      - ${{ parameters. preSteps }}
+      - ${{ parameters.preSteps }}
       - script: |
           if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
@@ -36,6 +36,7 @@ jobs:
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
+        condition: eq(variables.SKIP_CHECKOUT, '') 
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)

.azure-pipelines/docker-sonic-slave.yml

@@ -46,9 +46,9 @@ parameters:
   default: sonicdev
 
 variables:
-- ${{ if and(startsWith(variables['Build.SourceBranchName'], '202'), eq(length(variables['Build.SourceBranchName']), 6)) }}:
+- ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
   - name: BUILD_OPTIONS
-    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+    value: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
 
 stages:
 - stage: Build

.azure-pipelines/official-build-cisco-8000.yml

@@ -0,0 +1,64 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+schedules:
+- cron: "0 8 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr: none
+
+resources:
+  repositories:
+  - repository: cisco-8000
+    type: github
+    name: Cisco-8000-sonic/platform-cisco-8000
+    endpoint: cisco-connection
+
+stages:
+- stage: Build
+  pool: sonic
+  variables:
+    CACHE_MODE: wcache
+    SKIP_CHECKOUT: true
+    TERM: ''
+
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) ${{ variables.VERSION_CONTROL_OPTIONS }}'
+      preSteps:
+      - checkout: self
+        submodules: recursive
+        path: s
+        displayName: 'Checkout code'
+      - checkout: cisco-8000
+        submodules: recursive
+        path: s/platform/cisco-8000
+        displayName: 'Checkout cisco-8000'
+      - script: |
+          cisco_ref=$(cat platform/checkout/cisco-8000.ini | grep '^ref=' | cut -d= -f2)
+          echo "The cisco 8000 release version is $cisco_ref"
+          echo "##vso[task.setvariable variable=cisco.ref]$cisco_ref"
+        displayName: 'Export cisco release version'
+      - task: DownloadGitHubRelease@0
+        inputs:
+          connection: cisco-connection
+          userRepository: Cisco-8000-sonic/platform-cisco-8000
+          defaultVersionType: specificTag
+          version: $(cisco.ref)
+          itemPattern: 'artifactory-*.tar.gz'
+          downloadPath: '$(System.ArtifactsDirectory)'
+      - script: |
+          make PLATFORM=cisco-8000 platform/cisco-8000
+          tar xfz $(System.ArtifactsDirectory)/artifactory-*.tar.gz -C platform/cisco-8000
+        displayName: 'Setup cisco artifacts'
+      jobGroups:
+      - name: cisco-8000

.azure-pipelines/official-build-multi-asic.yml

@@ -0,0 +1,51 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+schedules:
+- cron: "0 18 * * Sun,Wed,Fri"
+  displayName: "Alternate day build"
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+  pool: sonicbld
+  variables:
+    CACHE_MODE: wcache
+    ${{ if eq(variables['Build.SourceBranchName'], '202012') }}:
+      VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web'
+  jobs:
+  - template: azure-pipelines-build.yml
+    parameters:
+      buildOptions: 'USERNAME=admin SONIC_BUILD_JOBS=$(nproc) BUILD_MULTIASIC_KVM=y ${{ variables.VERSION_CONTROL_OPTIONS }}'
+      jobGroups:
+      - name: vs
+
+- stage: Test
+  variables:
+  - name: inventory
+    value: veos_vtb
+  - name: testbed_file
+    value: vtestbed.csv
+
+  jobs:
+  - job:
+    pool: sonictest
+    displayName: "kvmtest-multi-asic-t1-lag"
+    timeoutInMinutes: 240
+
+    steps:
+    - template: run-test-template.yml
+      parameters:
+        dut: vlab-08
+        tbname: vms-kvm-four-asic-t1-lag
+        ptf_name: ptf_vms6-4
+        tbtype: multi-asic-t1-lag
+        image: sonic-4asic-vs.img.gz

.azure-pipelines/run-test-template.yml

@@ -7,6 +7,13 @@ parameters:
   type: string
 - name: ptf_name
   type: string
+- name: image
+  type: string
+  default: sonic-vs.img.gz
+  values:
+  - sonic-vs.img.gz
+  - sonic-4asic-vs.img.gz
+  - sonic-6asic-vs.img.gz
 
 steps:
 - checkout: self
@@ -21,7 +28,7 @@ steps:
 - script: |
     set -x
     sudo mkdir -p /data/sonic-vm/images
-    sudo cp -v ../target/sonic-vs.img.gz /data/sonic-vm/images/sonic-vs.img.gz
+    sudo cp -v ../target/${{ parameters.image}} /data/sonic-vm/images/sonic-vs.img.gz
     sudo gzip -fd /data/sonic-vm/images/sonic-vs.img.gz
     username=$(id -un)
     sudo chown -R $username.$username /data/sonic-vm

.gitignore

@@ -82,3 +82,14 @@ files/build/tmp
 dockers/**/buildinfo
 platform/**/buildinfo
 sonic-slave*/**/buildinfo
+
+# pytest coverage files
+.coverage
+coverage.xml
+test-results.xml
+htmlcov/
+
+# Dev tools
+.vscode/
+.idea/
+

.gitmodules

@@ -91,3 +91,9 @@
 	path = platform/broadcom/saibcm-modules-dnx
 	url = https://github.com/Azure/saibcm-modules.git
 	branch =  sdk-6.5.22-gpl-dnx
+[submodule "platform/broadcom/sonic-platform-modules-nokia"]
+	path = platform/broadcom/sonic-platform-modules-nokia
+	url = https://github.com/nokia/sonic-platform.git
+[submodule "src/linkmgrd"]
+	path = src/linkmgrd
+	url = https://github.com/Azure/sonic-linkmgrd.git

Makefile.cache

@@ -182,7 +182,10 @@ define GET_MOD_DEP_SHA
 	$(eval $(1)_DEP_FILES_MISSING := $(filter-out $(wildcard $($(1)_DEP_MOD_SHA_FILES)),$($(1)_DEP_MOD_SHA_FILES)) )
 	$(if $($(1)_DEP_FILES_MISSING), $(warning "[ DPKG ] Dependecy file(s) are not found for $(1) : $($(1)_DEP_FILES_MISSING)))
 
-	$(eval $(1)_DEP_MOD_SHA := $(shell git hash-object  $($(1)_DEP_MOD_SHA_FILES) \
+	# Include package dependencies hash values into package hash calculation
+	$(eval $(1)_DEP_PKGS_SHA := $(foreach dfile,$(1)_MOD_DEP_PKGS,$(dfile)_DEP_MOD_SHA $(dfile)_MOD_HASH))
+
+	$(eval $(1)_DEP_MOD_SHA := $(shell bash -c "git hash-object  $($(1)_DEP_MOD_SHA_FILES) && echo $($(1)_DEP_PKGS_SHA)" \
                             | sha1sum | awk '{print substr($$1,0,23);}'))
 endef
 

Makefile.work

@@ -10,6 +10,7 @@
 #  * ENABLE_ZTP: Enables zero touch provisioning.
 #  * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
 #  * INCLUDE_KUBERNETES: Allows including Kubernetes
+#  * INCLUDE_MUX: Include MUX feature/services for TOR switch.
 #  * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
 #  * by default for TOR switch.
 #  * ENABLE_SYNCD_RPC: Enables rpc-based syncd builds.
@@ -47,12 +48,13 @@ SHELL = /bin/bash
 USER := $(shell id -un)
 PWD := $(shell pwd)
 USER_LC := $(shell echo $(USER) | tr A-Z a-z)
+DOCKER_MACHINE := $(shell docker run --rm debian:buster uname -m)
 
 comma := ,
 
-ifeq ($(shell uname -m), aarch64)
+ifeq ($(DOCKER_MACHINE), aarch64)
 COMPILE_HOST_ARCH := arm64
-else ifeq ($(shell uname -m), armv7l)
+else ifeq ($(shell echo $(DOCKER_MACHINE) | grep -qE "armv7l|armv8l" && echo y),y)
 COMPILE_HOST_ARCH := armhf
 else
 COMPILE_HOST_ARCH := amd64
@@ -273,20 +275,22 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
                            INCLUDE_DHCP_RELAY=$(INCLUDE_DHCP_RELAY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
+                           SONIC_INCLUDE_MUX=$(INCLUDE_MUX) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
+                           BUILD_MULTIASIC_KVM=$(BUILD_MULTIASIC_KVM) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 .DEFAULT_GOAL :=  all
 
 %::
-ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 	@$(DOCKER_MULTIARCH_CHECK)
 ifneq ($(BLDENV), )
 	@$(DOCKER_SERVICE_MULTIARCH_CHECK)
@@ -361,7 +365,7 @@ reset :
 	    if [ $$ans == y ]; then
 	        echo "Resetting local repository. Please wait...";
 	        $(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) sudo rm -rf fsroot;
-	        if [[ "$(CONFIGURED_ARCH)" == "armhf" || "$(CONFIGURED_ARCH)" ==  "arm64" ]]; then
+	        if [ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && [[ "$(CONFIGURED_ARCH)" == "armhf" || "$(CONFIGURED_ARCH)" ==  "arm64" ]]; then
 	            echo "Stopping march $(CONFIGURED_ARCH) docker"
 	            sudo kill -9 `sudo cat /var/run/march/docker.pid` || true
 	            sudo rm -f /var/run/march/docker.pid || true

README.md

@@ -67,24 +67,27 @@
 Following is the instruction on how to build an [(ONIE)](https://github.com/opencomputeproject/onie) compatible network operating system (NOS) installer image for network switches, and also how to build docker images running inside the NOS. Note that SONiC image are build per ASIC platform. Switches using the same ASIC platform share a common image. For a list of supported switches and ASIC, please refer to this [list](https://github.com/Azure/SONiC/wiki/Supported-Devices-and-Platforms)
 
 # Hardware
-Any server can be a build image server. We are using a server with 1T hard disk. The OS is Ubuntu 16.04.
 
-## Prerequisites
+Any server can be a build image server as long as it has:
 
-Install pip and jinja in host build machine, execute below commands if j2/j2cli is not available:
+  * Multiple cores to increase build speed
+  * Plenty of RAM (less than 8 GiB is likely to cause issues)
+  * 300G of free disk space
 
-    sudo apt-get install -y python-pip
-    sudo python2 -m pip install -U pip==9.0.3
-    sudo pip install --force-reinstall --upgrade jinja2>=2.10
-    sudo pip install j2cli
+A good choice of OS for building SONiC is currently Ubuntu 20.04.
 
-Configure your system to allow running the 'docker' command without 'sudo':
-    Add current user to the docker group
-	 `sudo gpasswd -a ${USER} docker`
-    Log out and log back in so that your group membership is re-evaluated
+## Prerequisites
 
-## SAI Version
-Please refer to [SONiC roadmap](https://github.com/Azure/SONiC/wiki/Sonic-Roadmap-Planning) on the SAI version for each SONiC release.
+ * Install pip and jinja in host build machine, execute below commands if j2/j2cli is not available:
+
+```
+sudo apt install -y python3-pip
+sudo pip3 install j2cli
+```
+
+ * Install [Docker](https://docs.docker.com/engine/install/) and configure your system to allow running the 'docker' command without 'sudo':
+    * Add current user to the docker group: `sudo gpasswd -a ${USER} docker`
+    * Log out and log back in so that your group membership is re-evaluated
 
 ## Clone or fetch the code repository with all git submodules
 To clone the code repository recursively, assuming git version 1.9 or newer:
@@ -110,8 +113,22 @@ To build SONiC installer image and docker images, run the following commands:
     # Execute make configure once to configure ASIC
     make configure PLATFORM=[ASIC_VENDOR]
 
-    # Build SONiC image
-    make all
+    # Build SONiC image with 4 jobs in parallel.
+    # Note: You can set this higher, but 4 is a good number for most cases
+    # and is well-tested.
+    make SONIC_BUILD_JOBS=4 all
+
+ The supported ASIC vendors are:
+
+- PLATFORM=broadcom
+- PLATFORM=marvell
+- PLATFORM=mellanox
+- PLATFORM=cavium
+- PLATFORM=centec
+- PLATFORM=nephos
+- PLATFORM=innovium
+- PLATFORM=p4
+- PLATFORM=vs
 
 ## Usage for ARM Architecture
 To build Arm32 bit for (ARMHF) platform
@@ -147,7 +164,7 @@ To build Arm64 bit for platform
 
  **NOTE**:
 
-- Recommend reserving 50G free space to build one platform.
+- Recommend reserving at least 100G free space to build one platform with a single job. The build process will use more disk if you are setting `SONIC_BUILD_JOBS` to more than 1.
 - If Docker's workspace folder, `/var/lib/docker`, resides on a partition without sufficient free space, you may encounter an error like the following during a Docker container build job:
 
     `/usr/bin/tar: /path/to/sonic-buildimage/<some_file>: Cannot write: No space left on device`
@@ -156,17 +173,7 @@ To build Arm64 bit for platform
 - Use `http_proxy=[your_proxy] https_proxy=[your_proxy] no_proxy=[your_no_proxy] make` to enable http(s) proxy in the build process.
 - Add your user account to `docker` group and use your user account to make. `root` or `sudo` are not supported.
 
-The SONiC installer contains all docker images needed. SONiC uses one image for all devices of a same ASIC vendor. The supported ASIC vendors are:
-
-- PLATFORM=broadcom
-- PLATFORM=marvell
-- PLATFORM=mellanox
-- PLATFORM=cavium
-- PLATFORM=centec
-- PLATFORM=nephos
-- PLATFORM=innovium
-- PLATFORM=p4
-- PLATFORM=vs
+The SONiC installer contains all docker images needed. SONiC uses one image for all devices of a same ASIC vendor.
 
 For Broadcom ASIC, we build ONIE and EOS image. EOS image is used for Arista devices, ONIE image is used for all other Broadcom ASIC based devices.
 
@@ -198,6 +205,9 @@ It is recommended to use clean targets to clean all packages that are built toge
 ## Build debug dockers and debug SONiC installer image:
 SONiC build system supports building dockers and ONIE-image with debug tools and debug symbols, to help with live & core debugging. For details refer to [(SONiC Buildimage Guide)](https://github.com/Azure/sonic-buildimage/blob/master/README.buildsystem.md).
 
+## SAI Version
+Please refer to [SONiC roadmap](https://github.com/Azure/SONiC/wiki/Sonic-Roadmap-Planning) on the SAI version for each SONiC release.
+
 ## Notes:
 - If you are running make for the first time, a sonic-slave-${USER} docker image will be built automatically.
 This may take a while, but it is a one-time action, so please be patient.