Skip to content

Commit

Permalink
[build]: fix dpkg admindir corruption issue in parallel build (sonic-…
Browse files Browse the repository at this point in the history
…net#6408)

Fix #119

when parallel build is enable, multiple dpkg-buildpackage
instances are running at the same time. /var/lib/dpkg is shared
by all instances and the /var/lib/dpkg/updates could be corrupted
and cause the build failure.

the fix is to use overlay fs to mount separate /var/lib/dpkg
for each dpkg-buildpackage instance so that they are not affecting
each other.

Signed-off-by: Guohan Lu <[email protected]>
  • Loading branch information
lguohan committed Jan 21, 2021
1 parent 7e282b8 commit 63a044a
Show file tree
Hide file tree
Showing 23 changed files with 41 additions and 25 deletions.
12 changes: 12 additions & 0 deletions rules/functions
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,15 @@ $(foreach deb,$(1), \
fi; \
done; } )
endef

###############################################################################
## Setup overlay fs for dpkg admin directory /var/lib/dpkg
###############################################################################
define SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR
upperdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
workdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
mergedir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workdir=$$workdir $$mergedir
export SONIC_DPKG_ADMINDIR=$$mergedir
trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT
endef
8 changes: 6 additions & 2 deletions slave.mk
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ PYTHON_WHEELS_PATH = $(TARGET_PATH)/python-wheels
PROJECT_ROOT = $(shell pwd)
DBG_IMAGE_MARK = dbg
DBG_SRC_ARCHIVE_FILE = $(TARGET_PATH)/sonic_src.tar.gz
DPKG_ADMINDIR_PATH = /sonic/dpkg

CONFIGURED_PLATFORM := $(shell [ -f .platform ] && cat .platform || echo generic)
PLATFORM_PATH = platform/$(CONFIGURED_PLATFORM)
Expand All @@ -52,6 +53,7 @@ configure :
@mkdir -p target/debs
@mkdir -p target/files
@mkdir -p target/python-wheels
@mkdir -p $(DPKG_ADMINDIR_PATH)
@echo $(PLATFORM) > .platform

distclean : .platform clean
Expand Down Expand Up @@ -280,6 +282,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS))
# Apply series of patches if exist
if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && QUILT_PATCHES=../$(notdir $($*_SRC_PATH)).patch quilt push -a; popd; fi
$(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR)
# Build project and take package
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG)
# Clean up
Expand Down Expand Up @@ -307,9 +310,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
# Build project
pushd $($*_SRC_PATH) $(LOG)
[ ! -f ./autogen.sh ] || ./autogen.sh $(LOG)
$(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR)
$(if $($*_DPKG_TARGET),
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) $(LOG),
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) $(LOG)
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG),
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG)
)
popd $(LOG)
# Clean up
Expand Down
2 changes: 1 addition & 1 deletion src/bash/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
dget -u https://launchpad.net/debian/+archive/primary/+sourcefiles/bash/$(BASH_VERSION_FULL)/bash_$(BASH_VERSION_FULL).dsc

pushd bash-$(BASH_VERSION_MAJOR)
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $* $(DEST)/
2 changes: 1 addition & 1 deletion src/hiredis/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :

dpkg-source -x hiredis_$(HIREDIS_VERSION_FULL).dsc
pushd hiredis-$(HIREDIS_VERSION)
fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary
dpkg-buildpackage -rfakeroot -d -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $* $(DERIVED_TARGETS) $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/initramfs-tools/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :

# Build the package
rm -f debian/*.debhelper.log
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/iproute2/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
dpkg-source -x iproute2_$(IPROUTE2_VERSION_FULL).dsc

pushd iproute2-$(IPROUTE2_VERSION)
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $* $(DEST)/
2 changes: 1 addition & 1 deletion src/isc-dhcp/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
stg import -s ../patch/series

# Build source and Debian packages
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb packages to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/libnl3/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
dpkg-source -x libnl3_$(LIBNL3_VERSION).dsc

pushd ./libnl3-$(LIBNL3_VERSION_BASE)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/libteam/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
mv tmp/debian libteam/
rm -rf tmp
pushd ./libteam
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/lldpd/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
stg import -s ../patch/series

# Build source and Debian packages
env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb packages to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/monit/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
stg import -s ../patch/series

# Build source and Debian packages
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb packages to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/mpdecimal/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :

dpkg-source -x mpdecimal_$(MPDECIMAL_VERSION_FULL).dsc
pushd mpdecimal-$(MPDECIMAL_VERSION)
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $* $(DERIVED_TARGETS) $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/python-click/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
git reset --hard debian/$(PYTHON_CLICK_VERSION)

# Build source and Debian packages
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb package to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/python3/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
fi
done

dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

cp $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/radvd/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
stg import -s ../patch/series

# Build source and Debian packages
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb packages to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/redis/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
dpkg-source -x redis_$(REDIS_VERSION_FULL).dsc

pushd redis-$(REDIS_VERSION)
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/snmpd/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
stg init
stg import -s ../patch-$(SNMPD_VERSION)/series

fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary
dpkg-buildpackage -rfakeroot -b -d -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/socat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
# Build source and Debian packages
pushd socat-1.7.3.1
patch -p0 < ../enable_readline.patch
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

# Move the newly-built .deb packages to the destination directory
Expand Down
2 changes: 1 addition & 1 deletion src/sonic-device-data/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
done;

# Build the package
dpkg-buildpackage -rfakeroot -b -us -uc
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)

popd

Expand Down
4 changes: 2 additions & 2 deletions src/sonic-frr/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
cp -a debianpkg debian
make -f debian/rules backports

# new directory to build the package
# new directory to build the package
rm -rf frrpkg
mkdir frrpkg
cd frrpkg
Expand All @@ -34,7 +34,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
tar xf ../../frr_*sonic.debian8*.debian.tar.xz

# build package
dpkg-buildpackage -rfakeroot -b -us -uc
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
cd ..
mv $(DERIVED_TARGET) $* $(DEST)/

Expand Down
2 changes: 1 addition & 1 deletion src/tacacs/nss/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
git am ../0004-Skip-accessing-tacacs-servers-for-local-non-tacacs-u.patch
git am ../0005-libnss-Modify-parsing-of-IP-addr-and-port-number-str.patch

dpkg-buildpackage -rfakeroot -b -us -uc
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
4 changes: 2 additions & 2 deletions src/tacacs/pam/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ DERIVED_TARGETS = libtac2_$(PAM_TACPLUS_VERSION)_amd64.deb \
$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
# Obtain pam_tacplus
rm -rf ./pam_tacplus
git clone https://github.com/jeroennijhof/pam_tacplus.git
git clone https://github.com/jeroennijhof/pam_tacplus.git
pushd ./pam_tacplus
git checkout -f v1.4.1

Expand All @@ -20,7 +20,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
git apply ../0004-management-vrf-support.patch
git apply ../0005-pam-Modify-parsing-of-IP-address-and-port-number-to-.patch

dpkg-buildpackage -rfakeroot -b -us -uc
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down
2 changes: 1 addition & 1 deletion src/thrift/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
dpkg-source -x thrift_$(THRIFT_VERSION_FULL).dsc
pushd thrift-$(THRIFT_VERSION)
patch -p1 < ../patch/THRIFT-3577-assertion-failed.patch
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
popd

mv $(DERIVED_TARGETS) $* $(DEST)/
Expand Down

0 comments on commit 63a044a

Please sign in to comment.