Skip to content

Commit

Permalink
[ci] Paralize azure pipeline (#1040)
Browse files Browse the repository at this point in the history
1. Setup pipeline without manual effort when checkout new release branch.
2. Use correct branch when downloading artifacts or checkout relative repos.
3. Clear downloaded artifacts to avoid using outdated dependencies.
4. Use commonlib pipeline to download libnl3 and libyang instead of vs image build, to increase success rate.
5. Add weekly build to keep artifacts remaining.
  • Loading branch information
liushilongbuaa authored May 18, 2022
1 parent 57ed180 commit 564bea7
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 69 deletions.
25 changes: 17 additions & 8 deletions .azure-pipelines/build-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,40 +32,48 @@ jobs:
vmImage: 'ubuntu-20.04'

steps:
- checkout: self
clean: true
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 9
pipeline: Azure.sonic-swss-common
artifact: ${{ parameters.swss_common_artifact_name }}
path: $(Build.ArtifactStagingDirectory)/download
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
runBranch: 'refs/heads/$(BUILD_BRANCH)'
displayName: "Download sonic swss common deb packages"
- task: DownloadPipelineArtifact@2
inputs:
artifact: ${{ parameters.sairedis_artifact_name }}
path: $(Build.ArtifactStagingDirectory)/download
displayName: "Download sonic sairedis deb packages"
- task: DownloadPipelineArtifact@2
inputs:
artifact: ${{ parameters.swss_artifact_name }}
path: $(Build.ArtifactStagingDirectory)/download
displayName: "Download sonic swss artifact"
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 142
pipeline: Azure.sonic-buildimage.official.vs
artifact: sonic-buildimage.vs
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download sonic buildimage"
runBranch: 'refs/heads/$(BUILD_BRANCH)'
path: $(Build.ArtifactStagingDirectory)/download
patterns: '**/target/docker-sonic-vs.gz'
displayName: "Download sonic-buildimage docker-sonic-vs"
- script: |
set -ex
echo $(Build.DefinitionName).$(Build.BuildNumber)
docker load < ../target/docker-sonic-vs.gz
docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
mkdir -p .azure-pipelines/docker-sonic-vs/debs
cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
pushd .azure-pipelines
Expand All @@ -74,7 +82,8 @@ jobs:
popd
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
displayName: "Build sonic-docker-vs"
rm -rf $(Build.ArtifactStagingDirectory)/download
displayName: "Build docker-sonic-vs"
- publish: $(Build.ArtifactStagingDirectory)/
artifact: ${{ parameters.artifact_name }}
Expand Down
80 changes: 49 additions & 31 deletions .azure-pipelines/build-swss-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ parameters:
- name: sonic_slave
type: string

- name: debian_version
type: string

- name: sairedis_artifact_name
type: string

Expand All @@ -38,13 +41,22 @@ jobs:
pool:
${{ if ne(parameters.pool, 'default') }}:
name: ${{ parameters.pool }}
${{ if eq(parameters.pool, 'default') }}:
${{ else }}:
vmImage: 'ubuntu-20.04'

container:
image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest

steps:
- checkout: sonic-swss
submodules: true
clean: true
- script: |
set -ex
git checkout $(BUILD_BRANCH)
git submodule update
git status
displayName: Set up sonic-swss branch
- script: |
sudo apt-get install -y libhiredis0.14 libhiredis-dev
sudo apt-get install -y libzmq5 libzmq3-dev
Expand All @@ -60,51 +72,57 @@ jobs:
inputs:
source: specific
project: build
pipeline: 9
pipeline: Azure.sonic-swss-common
artifact: ${{ parameters.swss_common_artifact_name }}
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
runBranch: 'refs/heads/$(BUILD_BRANCH)'
path: $(Build.ArtifactStagingDirectory)/download
patterns: |
libswsscommon_1.0.0_*.deb
libswsscommon-dev_1.0.0*.deb
displayName: "Download sonic swss common deb packages"
- task: DownloadPipelineArtifact@2
inputs:
artifact: ${{ parameters.sairedis_artifact_name }}
displayName: "Download sonic sairedis deb packages"
path: $(Build.ArtifactStagingDirectory)/download
patterns: |
libsaivs_*.deb
libsaivs-dev_*.deb
libsairedis_*.deb
libsairedis-dev_*.deb
libsaimetadata_*.deb
libsaimetadata-dev_*.deb
syncd-vs_*.deb
displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 1
artifact: sonic-buildimage.vs
pipeline: Azure.sonic-buildimage.common_libs
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download sonic buildimage"
runBranch: 'refs/heads/$(BUILD_BRANCH)'
path: $(Build.ArtifactStagingDirectory)/download
artifact: common-lib
patterns: |
target/debs/${{ parameters.debian_version }}/libnl-3*.deb
target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
target/debs/${{ parameters.debian_version }}/libnl-route*.deb
target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
displayName: "Download common libs"

- script: |
sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
sudo dpkg -i libsaivs_*.deb
sudo dpkg -i libsaivs-dev_*.deb
sudo dpkg -i libsairedis_*.deb
sudo dpkg -i libsairedis-dev_*.deb
sudo dpkg -i libsaimetadata_*.deb
sudo dpkg -i libsaimetadata-dev_*.deb
sudo dpkg -i syncd-vs_*.deb
workingDirectory: $(Pipeline.Workspace)
set -ex
sudo dpkg -i $(find ./download -name *.deb)
rm -rf download || true
workingDirectory: $(Build.ArtifactStagingDirectory)
displayName: "Install libnl3, sonic swss common, and sairedis"
- checkout: sonic-swss
path: s
submodules: true
- script: |
set -ex
rm ../*.deb || true
./autogen.sh
dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
dpkg-buildpackage -us -uc -b -j$(nproc)
mv ../*.deb $(Build.ArtifactStagingDirectory)
displayName: "Compile sonic swss"
- publish: $(System.DefaultWorkingDirectory)/
- publish: $(Build.ArtifactStagingDirectory)
artifact: ${{ parameters.artifact_name }}
displayName: "Archive swss debian packages"
21 changes: 12 additions & 9 deletions .azure-pipelines/build-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
pool:
${{ if ne(parameters.pool, 'default') }}:
name: ${{ parameters.pool }}
${{ if eq(parameters.pool, 'default') }}:
${{ else }}:
vmImage: 'ubuntu-20.04'

container:
Expand Down Expand Up @@ -97,22 +97,25 @@ jobs:
inputs:
source: specific
project: build
pipeline: 9
pipeline: Azure.sonic-swss-common
artifact: ${{ parameters.swss_common_artifact_name }}
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}'
runBranch: 'refs/heads/$(BUILD_BRANCH)'
path: $(Build.ArtifactStagingDirectory)/download
displayName: "Download sonic swss common deb packages"
- script: |
cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
workingDirectory: $(Pipeline.Workspace)
set -ex
sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb
sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
rm -rf download
workingDirectory: $(Build.ArtifactStagingDirectory)
displayName: "Install sonic swss Common"
- script: |
set -ex
rm ../*.deb || true
./autogen.sh
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs && cp ../*.deb .
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs
mv ../*.deb .
displayName: "Compile sonic sairedis with coverage enabled"
- script: |
sudo cp azsyslog.conf /etc/rsyslog.conf
Expand Down
35 changes: 21 additions & 14 deletions .azure-pipelines/test-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,33 +18,39 @@ jobs:
ls -A1 | xargs -I{} sudo rm -rf {}
displayName: "Clean workspace"
- checkout: self
clean: true
displayName: "Checkout sonic-sairedis"
- checkout: sonic-swss
clean: true
displayName: "Checkout sonic-swss"
- script: |
set -ex
cd sonic-swss
git checkout $(BUILD_BRANCH)
displayName: Set up sonic-swss branch
- task: DownloadPipelineArtifact@2
inputs:
artifact: docker-sonic-vs
displayName: "Download docker sonic vs image"

path: $(Build.ArtifactStagingDirectory)/download
displayName: "Download pre-stage built docker-sonic-vs"
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 9
pipeline: Azure.sonic-swss-common
artifact: sonic-swss-common.amd64.ubuntu20_04
path: $(Build.ArtifactStagingDirectory)/download
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
runBranch: 'refs/heads/$(BUILD_BRANCH)'
displayName: "Download sonic swss common deb packages"

- checkout: self
displayName: "Checkout sonic-sairedis"
- checkout: sonic-swss
displayName: "Checkout sonic-swss"

- script: |
set -x
set -ex
sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh
sudo apt-get install -y libhiredis0.14
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
# install packages for vs test
sudo apt-get install -y net-tools bridge-utils vlan
Expand All @@ -53,12 +59,13 @@ jobs:
displayName: "Install dependencies"
- script: |
set -x
sudo docker load -i ../docker-sonic-vs.gz
set -ex
sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
docker ps
ip netns list
pushd sonic-swss/tests
sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
rm -rf $(Build.ArtifactStagingDirectory)/download
displayName: "Run vs tests"
- task: PublishTestResults@2
Expand Down
45 changes: 38 additions & 7 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,29 @@
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml

pr:
- master
- 202???
- 201???

trigger:
batch: true
branches:
include:
- "*"
- master
- 202???
- 201???

# this part need to be set in UI
schedules:
- cron: "0 0 * * 6"
displayName: Weekly build
branches:
include:
- master
- 202???
- 201???
always: true

resources:
repositories:
Expand All @@ -15,14 +34,25 @@ resources:
name: Azure/sonic-swss
endpoint: build

parameters:
- name: debian_version
type: string
default: buster
variables:
- name: BUILD_BRANCH
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
value: $(System.PullRequest.TargetBranch)
${{ else }}:
value: $(Build.SourceBranchName)

stages:
- stage: Build

jobs:
- template: .azure-pipelines/build-template.yml
parameters:
arch: amd64
sonic_slave: sonic-slave-buster
sonic_slave: sonic-slave-${{ parameters.debian_version }}
swss_common_artifact_name: sonic-swss-common
artifact_name: sonic-sairedis
syslog_artifact_name: sonic-sairedis.syslog
Expand All @@ -36,19 +66,19 @@ stages:
- template: .azure-pipelines/build-template.yml
parameters:
arch: armhf
timeout: 240
timeout: 180
pool: sonicbld-armhf
sonic_slave: sonic-slave-buster-armhf
sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf
swss_common_artifact_name: sonic-swss-common.armhf
artifact_name: sonic-sairedis.armhf
syslog_artifact_name: sonic-sairedis.syslog.armhf

- template: .azure-pipelines/build-template.yml
parameters:
arch: arm64
timeout: 240
timeout: 180
pool: sonicbld-arm64
sonic_slave: sonic-slave-buster-arm64
sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64
swss_common_artifact_name: sonic-swss-common.arm64
artifact_name: sonic-sairedis.arm64
syslog_artifact_name: sonic-sairedis.syslog.arm64
Expand All @@ -60,10 +90,11 @@ stages:
- template: .azure-pipelines/build-swss-template.yml
parameters:
arch: amd64
sonic_slave: sonic-slave-buster
sonic_slave: sonic-slave-${{ parameters.debian_version }}
swss_common_artifact_name: sonic-swss-common
sairedis_artifact_name: sonic-sairedis
artifact_name: sonic-swss
debian_version: ${{ parameters.debian_version }}

- stage: BuildDocker
dependsOn: BuildSwss
Expand Down

0 comments on commit 564bea7

Please sign in to comment.