From d6a6d25fe122a87bf3dfdbbf8b816bff1c7ca422 Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Wed, 9 Nov 2022 08:09:53 +0800
Subject: [PATCH 1/6] [Build] Support j2 template for debian sources (#12557)

Why I did it
Unify the Debian mirror sources
Make easy to upgrade to the next Debian release, not source url code change required.
Support to customize the Debian mirror sources during the build
Relative issue: #12523
---
 .gitignore                                    |  4 +++
 Makefile.work                                 | 14 ++++++--
 build_debian.sh                               |  1 +
 dockers/docker-base-buster/Dockerfile.j2      |  8 +----
 dockers/docker-base-buster/sources.list       | 13 -------
 dockers/docker-base-buster/sources.list.arm64 | 11 ------
 dockers/docker-base-buster/sources.list.armhf | 11 ------
 dockers/docker-base-stretch/Dockerfile.j2     |  8 +----
 dockers/docker-base-stretch/sources.list      | 11 ------
 .../docker-base-stretch/sources.list.arm64    | 11 ------
 .../docker-base-stretch/sources.list.armhf    | 11 ------
 files/apt/sources.list.amd64                  | 13 -------
 files/apt/sources.list.arm64                  | 13 -------
 files/apt/sources.list.armhf                  | 13 -------
 files/apt/sources.list.j2                     | 20 +++++++++++
 scripts/build_mirror_config.sh                | 26 ++++++++++++++
 scripts/prepare_docker_buildinfo.sh           |  4 +++
 slave.mk                                      |  2 ++
 sonic-slave-buster/Dockerfile.j2              | 34 +-----------------
 sonic-slave-stretch/Dockerfile.j2             | 35 +------------------
 20 files changed, 72 insertions(+), 191 deletions(-)
 delete mode 100644 dockers/docker-base-buster/sources.list
 delete mode 100644 dockers/docker-base-buster/sources.list.arm64
 delete mode 100644 dockers/docker-base-buster/sources.list.armhf
 delete mode 100644 dockers/docker-base-stretch/sources.list
 delete mode 100644 dockers/docker-base-stretch/sources.list.arm64
 delete mode 100644 dockers/docker-base-stretch/sources.list.armhf
 delete mode 100644 files/apt/sources.list.amd64
 delete mode 100644 files/apt/sources.list.arm64
 delete mode 100644 files/apt/sources.list.armhf
 create mode 100644 files/apt/sources.list.j2
 create mode 100755 scripts/build_mirror_config.sh

diff --git a/.gitignore b/.gitignore
index ced7ef1651e2..0527f343eb68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,3 +82,7 @@ files/build/tmp
 dockers/**/buildinfo
 platform/**/buildinfo
 sonic-slave*/**/buildinfo
+
+# Debian mirror Sources
+sources.list.*
+!sources.list*.j2
diff --git a/Makefile.work b/Makefile.work
index ffb9c8dc5e4b..77ca50bb899b 100644
--- a/Makefile.work
+++ b/Makefile.work
@@ -134,6 +134,7 @@ SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
 # Generate the version control build info
 $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+    MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
@@ -282,13 +283,20 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
+                           MIRROR_URLS=$(MIRROR_URLS) \
+                           MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
+                           MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 .DEFAULT_GOAL :=  all
 
-%::
+export MIRROR_URLS
+export MIRROR_SECURITY_URLS
+export SONIC_VERSION_CONTROL_COMPONENTS
+
+%:: | sonic-build-hooks
 ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
 	@$(DOCKER_MULTIARCH_CHECK)
 ifneq ($(BLDENV), )
@@ -298,8 +306,6 @@ endif
 endif
 	@$(OVERLAY_MODULE_CHECK)
 
-	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
-	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
 	@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
 	    { [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
 	      $(DOCKER_BASE_PULL) && \
@@ -324,6 +330,8 @@ endif
 sonic-build-hooks:
 	@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
 	@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
+	@[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV)
+	@scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)
 
 sonic-slave-base-build : sonic-build-hooks
 	@$(OVERLAY_MODULE_CHECK)
diff --git a/build_debian.sh b/build_debian.sh
index c5e26a7c0c9f..e3a6680f052c 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -104,6 +104,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
 [ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/
 
 ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
+scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO 
 sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
 sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
 
diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2
index e16a0f677b07..3735374526e1 100644
--- a/dockers/docker-base-buster/Dockerfile.j2
+++ b/dockers/docker-base-buster/Dockerfile.j2
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Configure data sources for apt/dpkg
 COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
-{% if CONFIGURED_ARCH == "armhf" %}
-COPY ["sources.list.armhf", "/etc/apt/sources.list"]
-{% elif CONFIGURED_ARCH == "arm64" %}
-COPY ["sources.list.arm64", "/etc/apt/sources.list"]
-{% else %}
-COPY ["sources.list", "/etc/apt/sources.list"]
-{% endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
 
diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list
deleted file mode 100644
index 0eef72d9fa2d..000000000000
--- a/dockers/docker-base-buster/sources.list
+++ /dev/null
@@ -1,13 +0,0 @@
-## Debian mirror on Microsoft Azure
-## Ref: http://debian-archive.trafficmanager.net/
-
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free
-
-# Debian mirror supports multiple versions for a package
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free
diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64
deleted file mode 100644
index 6375734e99e6..000000000000
--- a/dockers/docker-base-buster/sources.list.arm64
+++ /dev/null
@@ -1,11 +0,0 @@
-## Debian mirror for ARM repo
-
-# ARM repo
-deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
-deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
-deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free
-deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free
-deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free
diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf
deleted file mode 100644
index a03af1a33ac0..000000000000
--- a/dockers/docker-base-buster/sources.list.armhf
+++ /dev/null
@@ -1,11 +0,0 @@
-## Debian mirror for ARM repo
-
-# ARM repo
-deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
-deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
-deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free
-deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free
-deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free
diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2
index 5603dc502792..9e11c4ea35f0 100644
--- a/dockers/docker-base-stretch/Dockerfile.j2
+++ b/dockers/docker-base-stretch/Dockerfile.j2
@@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Configure data sources for apt/dpkg
 COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
-{% if CONFIGURED_ARCH == "armhf" %}
-COPY ["sources.list.armhf", "/etc/apt/sources.list"]
-{% elif CONFIGURED_ARCH == "arm64" %}
-COPY ["sources.list.arm64", "/etc/apt/sources.list"]
-{% else %}
-COPY ["sources.list", "/etc/apt/sources.list"]
-{% endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
 
diff --git a/dockers/docker-base-stretch/sources.list b/dockers/docker-base-stretch/sources.list
deleted file mode 100644
index 0c29b339bb87..000000000000
--- a/dockers/docker-base-stretch/sources.list
+++ /dev/null
@@ -1,11 +0,0 @@
-## Debian mirror on Microsoft Azure
-## Ref: http://debian-archive.trafficmanager.net/
-
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch-backports main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free
diff --git a/dockers/docker-base-stretch/sources.list.arm64 b/dockers/docker-base-stretch/sources.list.arm64
deleted file mode 100644
index 520c46519919..000000000000
--- a/dockers/docker-base-stretch/sources.list.arm64
+++ /dev/null
@@ -1,11 +0,0 @@
-## Debian mirror for ARM repo
-
-# ARM repo
-deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free
-deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free
-deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free
-deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free
-deb [arch=arm64] http://deb.debian.org/debian/ stretch-backports main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free
diff --git a/dockers/docker-base-stretch/sources.list.armhf b/dockers/docker-base-stretch/sources.list.armhf
deleted file mode 100644
index 58077f310424..000000000000
--- a/dockers/docker-base-stretch/sources.list.armhf
+++ /dev/null
@@ -1,11 +0,0 @@
-## Debian mirror for ARM repo
-
-# ARM repo
-deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free
-deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free
-deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free
-deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free
-deb [arch=armhf] http://deb.debian.org/debian/ stretch-backports main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free
diff --git a/files/apt/sources.list.amd64 b/files/apt/sources.list.amd64
deleted file mode 100644
index 45902be1078c..000000000000
--- a/files/apt/sources.list.amd64
+++ /dev/null
@@ -1,13 +0,0 @@
-## Debian mirror on Microsoft Azure
-## Ref: http://debian-archive.trafficmanager.net/
-
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
-deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
-deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ buster_updates main contrib non-free
-deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ buster_updates main contrib non-free
-deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free
diff --git a/files/apt/sources.list.arm64 b/files/apt/sources.list.arm64
deleted file mode 100644
index 58b84978d023..000000000000
--- a/files/apt/sources.list.arm64
+++ /dev/null
@@ -1,13 +0,0 @@
-## Debian mirror for ARM
-## Not the repo mirror site can change in future, and needs to be updated to be in sync
-
-deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
-deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
-deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free
-deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free
-deb [arch=arm64] http://ftp.debian.org/debian buster-backports main
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main
diff --git a/files/apt/sources.list.armhf b/files/apt/sources.list.armhf
deleted file mode 100644
index eb6fe3be6889..000000000000
--- a/files/apt/sources.list.armhf
+++ /dev/null
@@ -1,13 +0,0 @@
-## Debian mirror for ARM
-## Not the repo mirror site can change in future, and needs to be updated to be in sync
-
-deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
-deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
-deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free
-deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free
-deb [arch=armhf] http://ftp.debian.org/debian buster-backports main
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free
-deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main
diff --git a/files/apt/sources.list.j2 b/files/apt/sources.list.j2
new file mode 100644
index 000000000000..57d53b919b03
--- /dev/null
+++ b/files/apt/sources.list.j2
@@ -0,0 +1,20 @@
+# The configuration is generated by template
+# Please add additional sources in /etc/apt/sources.list.d
+
+{% for mirror_url in MIRROR_URLS.split(',') %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free
+{% endfor %}
+{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %}
+{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %}
+{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
+{% else %}
+deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
+deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
+{% endif %}
+{% endfor %}
diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh
new file mode 100755
index 000000000000..5f94e701cbde
--- /dev/null
+++ b/scripts/build_mirror_config.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Generate the sources.list.<arch> in the config path
+CONFIG_PATH=$1
+export ARCHITECTURE=$2
+export DISTRIBUTION=$3
+
+# The default mirror urls
+DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
+DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
+
+# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
+if [ "$ARCHITECTURE" == "armhf" ]; then
+    DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
+    DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
+fi
+
+[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
+[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
+
+TEMPLATE=files/apt/sources.list.j2
+[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2
+[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2
+[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
+
+MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh
index d2b2c57185d1..3c1104eddc53 100755
--- a/scripts/prepare_docker_buildinfo.sh
+++ b/scripts/prepare_docker_buildinfo.sh
@@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then
     [ -z "$DISTRO" ] && DISTRO=jessie
 fi
 
+if [[ "$IMAGENAME" == docker-base-* ]]; then
+    scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
+fi
+
 # add script for reproducible build. using sha256 instead of tag for docker base image.
 scripts/docker_version_control.sh $@
 
diff --git a/slave.mk b/slave.mk
index b8148e4bcfba..9f11cbe0e9e4 100644
--- a/slave.mk
+++ b/slave.mk
@@ -121,6 +121,8 @@ export TRUSTED_GPG_URLS
 export SONIC_VERSION_CONTROL_COMPONENTS
 DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY)
 export DEFAULT_CONTAINER_REGISTRY
+export MIRROR_URLS
+export MIRROR_SECURITY_URLS
 
 ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y)
 ENABLE_PFCWD_ON_START = y
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index de5f9911b6f0..3747f72f5050 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -12,39 +12,7 @@ FROM {{ prefix }}debian:buster
 MAINTAINER gulv@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
-
-RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
-
-{%- if CONFIGURED_ARCH == "armhf" %}
-RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
-{%- elif CONFIGURED_ARCH == "arm64" %}
-RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
-{%- endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2
index f2c4751351d0..975c2f1989c0 100644
--- a/sonic-slave-stretch/Dockerfile.j2
+++ b/sonic-slave-stretch/Dockerfile.j2
@@ -10,40 +10,7 @@ FROM {{ prefix }}debian:stretch
 MAINTAINER gulv@microsoft.com
 
 COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
-
-RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=amd64] http://packages.microsoft.com/debian/9/prod stretch main" >> /etc/apt/sources.list
-
-{%- if CONFIGURED_ARCH == "armhf" %}
-RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list
-{%- elif CONFIGURED_ARCH == "arm64" %}
-RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \
-        echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list
-{%- endif %}
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive

From 624e48ea413f0d23f47743c7f745fc1d1d3e96bb Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Fri, 13 Jan 2023 16:16:35 +0800
Subject: [PATCH 2/6] [Build] Support Debian snapshot mirror to improve build
 stability (#13097)

Why I did it
[Build] Support Debian snapshot mirror to improve build stability

It is to enhance the reproducible build, supports the Debian snapshot mirror. It guarantees all the docker images using the same Debian mirror snapshot and fixes the temporary build failure which is caused by remote Debain mirror indexes changed during the build. It is also to fix the version conflict issue caused by no fixed versions of some of the Debian packages.

How I did it
Add a new feature to support the Debian snapshot mirror.

How to verify it
---
 rules/config                                  |  3 ++
 scripts/build_mirror_config.sh                | 28 +++++++++++++++++++
 scripts/generate_buildinfo_config.sh          |  1 +
 scripts/versions_manager.py                   |  9 ++++--
 slave.mk                                      |  1 +
 .../scripts/buildinfo_base.sh                 | 27 +++++++++++++++++-
 .../scripts/collect_version_files             | 10 +++++++
 7 files changed, 75 insertions(+), 4 deletions(-)

diff --git a/rules/config b/rules/config
index 92b2809200e4..49d44c99d33c 100644
--- a/rules/config
+++ b/rules/config
@@ -186,6 +186,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt
 #   docker: docker base images
 SONIC_VERSION_CONTROL_COMPONENTS ?= none
 
+# MIRROR_SNAPSHOT - support mirror snapshot flag
+MIRROR_SNAPSHOT ?= n
+
 # SONiC docker registry
 #
 # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry
diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh
index 5f94e701cbde..a1e5900a48b8 100755
--- a/scripts/build_mirror_config.sh
+++ b/scripts/build_mirror_config.sh
@@ -5,16 +5,40 @@ CONFIG_PATH=$1
 export ARCHITECTURE=$2
 export DISTRIBUTION=$3
 
+DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
+[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror
+
 # The default mirror urls
 DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
 DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
 
+
 # The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
 if [ "$ARCHITECTURE" == "armhf" ]; then
     DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
     DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
 fi
 
+if [ "$MIRROR_SNAPSHOT" == y ]; then
+    if [ -f $MIRROR_VERSION_FILE ]; then
+        DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
+        DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
+    elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+    fi
+
+    DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/
+    DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/
+
+    mkdir -p target/versions/default
+    if [ ! -f target/versions/default/versions-mirror ]; then
+        echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+        echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+    fi
+fi
+
 [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
 [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
 
@@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2
 [ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2
 
 MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
+if [ "$MIRROR_SNAPSHOT" == y ]; then
+    # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
+    sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
+fi
diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh
index fe7657a6b6c9..080e6a321a6b 100755
--- a/scripts/generate_buildinfo_config.sh
+++ b/scripts/generate_buildinfo_config.sh
@@ -8,3 +8,4 @@ mkdir -p $BUILDINFO_PATH/buildinfo/config
 
 echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG
 echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG
+echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG
diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py
index a20684e97bb4..d4127dece932 100755
--- a/scripts/versions_manager.py
+++ b/scripts/versions_manager.py
@@ -475,12 +475,13 @@ def filter(self, ctypes=[]):
             module.filter(ctypes=ctypes)
 
     def get_default_module(self):
-        if DEFAULT_MODULE in self.modules:
-            return self.modules[DEFAULT_MODULE]
+        default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, []))
         ctypes = self.get_component_types()
         dists = self.get_dists()
         components = []
         for ctype in ctypes:
+            if ctype in DEFAULT_OVERWRITE_COMPONENTS:
+                continue
             if ctype == 'deb':
                 for dist in dists:
                     versions = self._get_versions(ctype, dist)
@@ -492,7 +493,9 @@ def get_default_module(self):
                 common_versions = self._get_common_versions(versions)
                 component = Component(common_versions, ctype)
                 components.append(component)
-        return VersionModule(DEFAULT_MODULE, components)
+        module = VersionModule(DEFAULT_MODULE, components)
+        module.overwrite(default_module, True, True)
+        return module
 
     def get_aggregatable_modules(self):
         modules = {}
diff --git a/slave.mk b/slave.mk
index 9f11cbe0e9e4..865be99c9e78 100644
--- a/slave.mk
+++ b/slave.mk
@@ -71,6 +71,7 @@ export IMAGE_DISTRO
 export IMAGE_DISTRO_DEBS_PATH
 export MULTIARCH_QEMU_ENVIRON
 export DOCKER_BASE_ARCH
+export MIRROR_SNAPSHOT
 
 ###############################################################################
 ## Utility rules
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
index ff249dfdfb91..b9cbb5b3fbc7 100755
--- a/src/sonic-build-hooks/scripts/buildinfo_base.sh
+++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions
 VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb
 WEB_VERSION_FILE=$VERSION_PATH/versions-web
 BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
-REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian'
+REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
 DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
 
 . $BUILDINFO_PATH/config/buildinfo.config
@@ -72,14 +72,39 @@ set_reproducible_mirrors()
 {
     # Remove the charater # in front of the line if matched
     local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/"
+    # Add the character # in front of the line, if not match the URL pattern condition
+    local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/"
+    local expression3="\$a#SET_REPR_MIRRORS"
     if [ "$1" = "-d" ]; then
         # Add the charater # in front of the line if match
         expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/"
+        # Remove the character # in front of the line, if not match the URL pattern condition
+        expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/"
+        expression3="/#SET_REPR_MIRRORS/d"
     fi
 
     local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)"
     for mirror in $mirrors; do
+        if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then
+            continue
+        fi
+
+        # Make sure no duplicate operations on the mirror config file
+        if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") ||
+           ([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then
+            continue
+        fi
+
+        # Enable or disable the reproducible mirrors
         $SUDO sed -i "$expression" "$mirror"
+
+        # Enable or disable the none reproducible mirrors
+        if [ "$MIRROR_SNAPSHOT" == y ]; then
+            $SUDO sed -ri "$expression2" "$mirror"
+        fi
+
+        # Add or remove the SET_REPR_MIRRORS flag
+        $SUDO sed -i "$expression3" "$mirror"
     done
 }
 
diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files
index a4b33eeaa897..3fcc99a87106 100755
--- a/src/sonic-build-hooks/scripts/collect_version_files
+++ b/src/sonic-build-hooks/scripts/collect_version_files
@@ -18,6 +18,13 @@ dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DI
 ## Add the the packages purged
 [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
 
+## Add mirror versions
+while read -r line; do
+    mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//")
+    date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ)
+    echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror
+done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null)
+
 ## Print the unique and sorted result
 sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}"
 if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then
@@ -26,5 +33,8 @@ fi
 if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then
     sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}"
 fi
+if [ -e "${TARGET_PATH}/versions-mirror" ]; then
+    sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror"
+fi
 
 exit 0

From 15be08266e7980ee73037def40336e4ea124831c Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Tue, 14 Feb 2023 14:59:38 +0800
Subject: [PATCH 3/6] [Build] Change the default mirror version config file
 (#13786)

Why I did it
Change the mirror config file
Use the files/build/versions/default/versions-mirror only when reproducible build enabled.
The config in files/build/versions is only for reproducible build, while snapshot mirror feature does not have the dependency on the reproducible build.

How I did it
Skip the mirror config in files/build/versions/default/versions-mirror if reproducible build not enabled.

How to verify it
---
 scripts/build_mirror_config.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh
index a1e5900a48b8..aee56f23ae6a 100755
--- a/scripts/build_mirror_config.sh
+++ b/scripts/build_mirror_config.sh
@@ -6,7 +6,8 @@ export ARCHITECTURE=$2
 export DISTRIBUTION=$3
 
 DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
-MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
+MIRROR_VERSION_FILE=
+[[ "$SONIC_VERSION_CONTROL_COMPONENTS" == *deb* || $SONIC_VERSION_CONTROL_COMPONENTS == *all* ]] && MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
 [ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror
 
 # The default mirror urls
@@ -21,7 +22,7 @@ if [ "$ARCHITECTURE" == "armhf" ]; then
 fi
 
 if [ "$MIRROR_SNAPSHOT" == y ]; then
-    if [ -f $MIRROR_VERSION_FILE ]; then
+    if [ -f "$MIRROR_VERSION_FILE" ]; then
         DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
         DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//')
     elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then

From 8ab13bd815d602734b1d835a0fb2486a9f244c81 Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Wed, 22 Feb 2023 11:18:09 +0800
Subject: [PATCH 4/6] [Ci] Support to use the same snapshot for all platform
 builds (#13913)

IBGP
---
 .../azure-pipelines-UpgrateVersion.yml        | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/.azure-pipelines/azure-pipelines-UpgrateVersion.yml b/.azure-pipelines/azure-pipelines-UpgrateVersion.yml
index df8f7dac2a3a..51cb646eefbc 100644
--- a/.azure-pipelines/azure-pipelines-UpgrateVersion.yml
+++ b/.azure-pipelines/azure-pipelines-UpgrateVersion.yml
@@ -42,12 +42,32 @@ parameters:
   - mellanox
 
 stages:
+- stage: Prepare
+  jobs:
+  - job: Prepare
+    steps:
+    - script: |
+        DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
+        DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp)
+        DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp)
+        echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_TIMESTAMP;isOutput=true]$DEBIAN_TIMESTAMP"
+        echo "##vso[task.setvariable variable=DEBIAN_SECURITY_TIMESTAMP;isOutput=true]$DEBIAN_SECURITY_TIMESTAMP"
+      name: SetVersions
+      displayName: 'Set snapshot versions'
 - stage: Build
+  dependsOn: Prepare
   variables:
     - name: CACHE_MODE
       value: none
     - name: VERSION_CONTROL_OPTIONS
       value: 'SONIC_VERSION_CONTROL_COMPONENTS='
+    - name: SKIP_CHECKOUT
+      value: true
+    - name: DEBIAN_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_TIMESTAMP'] ]
+    - name: DEBIAN_SECURITY_TIMESTAMP
+      value: $[ stageDependencies.Prepare.Prepare.outputs['SetVersions.DEBIAN_SECURITY_TIMESTAMP'] ]
     - template: .azure-pipelines/template-variables.yml@buildimage
   jobs:
   - template: azure-pipelines-build.yml
@@ -56,6 +76,21 @@ stages:
       buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
       - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
+      - checkout: self
+        submodules: recursive
+        fetchDepth: 0
+        path: s
+        displayName: 'Checkout code'
+      - script: |
+          echo "DEBIAN_TIMESTAMP=$DEBIAN_TIMESTAMP, DEBIAN_SECURITY_TIMESTAMP=$DEBIAN_SECURITY_TIMESTAMP"
+          if [ "$MIRROR_SNAPSHOT" == y ]; then
+            mkdir -p target/versions/default/
+            echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror
+            echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror
+            cat target/versions/default/versions-mirror
+          fi
+        displayName: 'Set snapshot versions'
+
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

From 1b34b2aa7a6da4380583998e3f1ab833653ff609 Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Fri, 30 Dec 2022 15:33:21 +0800
Subject: [PATCH 5/6] [Build] Support j2 template for debian sources for docker
 ptf (#13198)

Change to use the sources.list from the file generated from the j2 template
---
 dockers/docker-ptf/Dockerfile.j2    | 8 +++-----
 scripts/prepare_docker_buildinfo.sh | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/dockers/docker-ptf/Dockerfile.j2 b/dockers/docker-ptf/Dockerfile.j2
index 9d0660b1592b..38bb262918bf 100644
--- a/dockers/docker-ptf/Dockerfile.j2
+++ b/dockers/docker-ptf/Dockerfile.j2
@@ -9,9 +9,6 @@ FROM {{ prefix }}debian:stretch
 
 MAINTAINER Pavel Shirshov
 
-## Remove retired jessie-updates repo
-RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
-
 ## Copy dependencies
 COPY \
 {% for deb in docker_ptf_debs.split(' ') -%}
@@ -19,13 +16,14 @@ debs/{{ deb }}{{' '}}
 {%- endfor -%}
 debs/
 
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
+
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
 ## Set the apt source, update package cache and install necessary packages
 ## TODO: Clean up this step
-RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /etc/apt/sources.list \
-    && apt-get update          \
+RUN apt-get update          \
     && apt-get upgrade -y   \
     && apt-get dist-upgrade -y  \
     && apt-get install -y   \
diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh
index 3c1104eddc53..c5547dfc3094 100755
--- a/scripts/prepare_docker_buildinfo.sh
+++ b/scripts/prepare_docker_buildinfo.sh
@@ -22,7 +22,7 @@ if [ -z "$DISTRO" ]; then
     [ -z "$DISTRO" ] && DISTRO=jessie
 fi
 
-if [[ "$IMAGENAME" == docker-base-* ]]; then
+if [[ "$IMAGENAME" == docker-base-* ]] || [[ "$IMAGENAME" == docker-ptf ]]; then
     scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
 fi
 

From b7ee985903e07a47fcb8f2e72fe1b1b237d0f008 Mon Sep 17 00:00:00 2001
From: xumia <59720581+xumia@users.noreply.github.com>
Date: Tue, 11 Apr 2023 17:07:26 +0800
Subject: [PATCH 6/6] [Build] Optimize the version control for Debian packages
 (#14557)

Why I did it
Optimize the version control for Debian packages.
Fix sonic-slave-buster/sources.list.amd64 not found display issue, need to generate the file before running the shell command to evaluate the sonic image tag.
When using the snapshot mirror, it is not necessary to update the version file based on the base image. It will reduce the version dependency issue, when an image is not run when freezing the version.

How I did it
Not to update the version file when snapshot mirror enabled.

How to verify it
---
 Makefile.work                                   | 2 +-
 scripts/prepare_docker_buildinfo.sh             | 2 +-
 src/sonic-build-hooks/scripts/buildinfo_base.sh | 3 +++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/Makefile.work b/Makefile.work
index 77ca50bb899b..86c4dee31a2d 100644
--- a/Makefile.work
+++ b/Makefile.work
@@ -140,7 +140,7 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
 # Generate the slave Dockerfile, and prepare build info for it
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
-$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh
index c5547dfc3094..f39fe711dd48 100755
--- a/scripts/prepare_docker_buildinfo.sh
+++ b/scripts/prepare_docker_buildinfo.sh
@@ -22,7 +22,7 @@ if [ -z "$DISTRO" ]; then
     [ -z "$DISTRO" ] && DISTRO=jessie
 fi
 
-if [[ "$IMAGENAME" == docker-base-* ]] || [[ "$IMAGENAME" == docker-ptf ]]; then
+if [[ "$IMAGENAME" == sonic-slave-* ]] || [[ "$IMAGENAME" == docker-base-* ]] || [[ "$IMAGENAME" == docker-ptf ]]; then
     scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
 fi
 
diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh
index b9cbb5b3fbc7..e57d362cbcec 100755
--- a/src/sonic-build-hooks/scripts/buildinfo_base.sh
+++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh
@@ -337,6 +337,9 @@ update_version_file()
 update_version_files()
 {
     local version_names="versions-deb versions-py2 versions-py3"
+    if [ "$MIRROR_SNAPSHOT" == y ]; then
+        version_names="versions-py2 versions-py3"
+    fi
     for version_name in $version_names; do
         update_version_file $version_name
     done