From 35d1b978815230c3b5316a19f2f2fa0dd6ddc54c Mon Sep 17 00:00:00 2001 From: xumia <59720581+xumia@users.noreply.github.com> Date: Fri, 13 Jan 2023 16:16:35 +0800 Subject: [PATCH] [Build][202211] Support Debian snapshot mirror to improve build stability --- Makefile.work | 2 ++ rules/config | 3 ++ scripts/build_mirror_config.sh | 28 +++++++++++++++++++ scripts/generate_buildinfo_config.sh | 1 + scripts/versions_manager.py | 9 ++++-- slave.mk | 1 + .../scripts/buildinfo_base.sh | 27 +++++++++++++++++- .../scripts/collect_version_files | 10 +++++++ 8 files changed, 77 insertions(+), 4 deletions(-) diff --git a/Makefile.work b/Makefile.work index adc40f381613..0a578c3e8672 100644 --- a/Makefile.work +++ b/Makefile.work @@ -180,6 +180,7 @@ $(shell \ SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \ scripts/generate_buildinfo_config.sh) # Generate the slave Dockerfile, and prepare build info for it @@ -501,6 +502,7 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \ SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \ MIRROR_URLS=$(MIRROR_URLS) \ MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \ + MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset diff --git a/rules/config b/rules/config index 9bb3326faa28..0ec0b575fea7 100644 --- a/rules/config +++ b/rules/config @@ -229,6 +229,9 @@ TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,htt # docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none +# MIRROR_SNAPSHOT - support mirror snapshot flag +MIRROR_SNAPSHOT ?= n + # SONiC docker registry # # Set the env variable ENABLE_DOCKER_BASE_PULL = y to enable pulling sonic-slave docker from registry diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh index 5f94e701cbde..a1e5900a48b8 100755 --- a/scripts/build_mirror_config.sh +++ b/scripts/build_mirror_config.sh @@ -5,16 +5,40 @@ CONFIG_PATH=$1 export ARCHITECTURE=$2 export DISTRIBUTION=$3 +DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net +MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror +[ -f target/versions/default/versions-mirror ] && MIRROR_VERSION_FILE=target/versions/default/versions-mirror + # The default mirror urls DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ + # The debian-archive.trafficmanager.net does not support armhf, use debian.org instead if [ "$ARCHITECTURE" == "armhf" ]; then DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/ DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ fi +if [ "$MIRROR_SNAPSHOT" == y ]; then + if [ -f $MIRROR_VERSION_FILE ]; then + DEBIAN_TIMESTAMP=$(grep "^debian==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + DEBIAN_SECURITY_TIMESTAMP=$(grep "^debian-security==" $MIRROR_VERSION_FILE | tail -n 1 | sed 's/.*==//') + elif [ -z "$DEBIAN_TIMESTAMP" ] || [ -z "$DEBIAN_SECURITY_TIMESTAMP" ]; then + DEBIAN_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian/latest/timestamp) + DEBIAN_SECURITY_TIMESTAMP=$(curl $DEFAULT_MIRROR_URL_PREFIX/snapshot/debian-security/latest/timestamp) + fi + + DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/snapshot/debian/$DEBIAN_TIMESTAMP/ + DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/snapshot/debian-security/$DEBIAN_SECURITY_TIMESTAMP/ + + mkdir -p target/versions/default + if [ ! -f target/versions/default/versions-mirror ]; then + echo "debian==$DEBIAN_TIMESTAMP" > target/versions/default/versions-mirror + echo "debian-security==$DEBIAN_SECURITY_TIMESTAMP" >> target/versions/default/versions-mirror + fi +fi + [ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS [ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS @@ -24,3 +48,7 @@ TEMPLATE=files/apt/sources.list.j2 [ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2 MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE +if [ "$MIRROR_SNAPSHOT" == y ]; then + # Set the snapshot mirror, and add the SET_REPR_MIRRORS flag + sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE +fi diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index fe7657a6b6c9..080e6a321a6b 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -8,3 +8,4 @@ mkdir -p $BUILDINFO_PATH/buildinfo/config echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG +echo "export MIRROR_SNAPSHOT=$MIRROR_SNAPSHOT" >> $BUILDINFO_CONFIG diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index a20684e97bb4..d4127dece932 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -475,12 +475,13 @@ def filter(self, ctypes=[]): module.filter(ctypes=ctypes) def get_default_module(self): - if DEFAULT_MODULE in self.modules: - return self.modules[DEFAULT_MODULE] + default_module = self.modules.get(DEFAULT_MODULE, VersionModule(DEFAULT_MODULE, [])) ctypes = self.get_component_types() dists = self.get_dists() components = [] for ctype in ctypes: + if ctype in DEFAULT_OVERWRITE_COMPONENTS: + continue if ctype == 'deb': for dist in dists: versions = self._get_versions(ctype, dist) @@ -492,7 +493,9 @@ def get_default_module(self): common_versions = self._get_common_versions(versions) component = Component(common_versions, ctype) components.append(component) - return VersionModule(DEFAULT_MODULE, components) + module = VersionModule(DEFAULT_MODULE, components) + module.overwrite(default_module, True, True) + return module def get_aggregatable_modules(self): modules = {} diff --git a/slave.mk b/slave.mk index 0cb2cd894491..9b1b799b846f 100644 --- a/slave.mk +++ b/slave.mk @@ -86,6 +86,7 @@ export DOCKER_BASE_ARCH export CROSS_BUILD_ENVIRON export BLDENV export BUILD_WORKDIR +export MIRROR_SNAPSHOT ############################################################################### ## Utility rules diff --git a/src/sonic-build-hooks/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh index ff249dfdfb91..b9cbb5b3fbc7 100755 --- a/src/sonic-build-hooks/scripts/buildinfo_base.sh +++ b/src/sonic-build-hooks/scripts/buildinfo_base.sh @@ -11,7 +11,7 @@ POST_VERSION_PATH=$BUILDINFO_PATH/post-versions VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb WEB_VERSION_FILE=$VERSION_PATH/versions-web BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web -REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/debian' +REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/' DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock . $BUILDINFO_PATH/config/buildinfo.config @@ -72,14 +72,39 @@ set_reproducible_mirrors() { # Remove the charater # in front of the line if matched local expression="s/^#\(.*$REPR_MIRROR_URL_PATTERN\)/\1/" + # Add the character # in front of the line, if not match the URL pattern condition + local expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#*deb/#&/" + local expression3="\$a#SET_REPR_MIRRORS" if [ "$1" = "-d" ]; then # Add the charater # in front of the line if match expression="s/^deb.*$REPR_MIRROR_URL_PATTERN/#\0/" + # Remove the character # in front of the line, if not match the URL pattern condition + expression2="/^#*deb.*$REPR_MIRROR_URL_PATTERN/! s/^#(#*deb)/\1/" + expression3="/#SET_REPR_MIRRORS/d" fi local mirrors="/etc/apt/sources.list $(find /etc/apt/sources.list.d/ -type f)" for mirror in $mirrors; do + if ! grep -iq "$REPR_MIRROR_URL_PATTERN" "$mirror"; then + continue + fi + + # Make sure no duplicate operations on the mirror config file + if ([ "$1" == "-d" ] && ! grep -iq "#SET_REPR_MIRRORS" "$mirror") || + ([ "$1" != "-d" ] && grep -iq "#SET_REPR_MIRRORS" "$mirror"); then + continue + fi + + # Enable or disable the reproducible mirrors $SUDO sed -i "$expression" "$mirror" + + # Enable or disable the none reproducible mirrors + if [ "$MIRROR_SNAPSHOT" == y ]; then + $SUDO sed -ri "$expression2" "$mirror" + fi + + # Add or remove the SET_REPR_MIRRORS flag + $SUDO sed -i "$expression3" "$mirror" done } diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files index a4b33eeaa897..3fcc99a87106 100755 --- a/src/sonic-build-hooks/scripts/collect_version_files +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -18,6 +18,13 @@ dpkg-query -W -f '${Package}==${Version}\n' >> "${TARGET_PATH}/versions-deb-${DI ## Add the the packages purged [ -f $POST_VERSION_PATH/purge-versions-deb ] && cat $POST_VERSION_PATH/purge-versions-deb >> "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" +## Add mirror versions +while read -r line; do + mirror=$(echo "$line" | sed "s/.*\///" | sed "s/_InRelease.*//") + date=$(date --date="$(echo "$line" | cut -d: -f3-)" +%Y-%m-%dT%H:%M:%SZ) + echo "$mirror==$date" >> ${TARGET_PATH}/versions-mirror +done < <(grep Date: /var/lib/apt/lists/*_InRelease 2>/dev/null) + ## Print the unique and sorted result sort -u "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" if [ -e "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ]; then @@ -26,5 +33,8 @@ fi if [ -e "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" ]; then sort -u "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" -o "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" fi +if [ -e "${TARGET_PATH}/versions-mirror" ]; then + sort -u "${TARGET_PATH}/versions-mirror" -o "${TARGET_PATH}/versions-mirror" +fi exit 0