feat(dmsquash-live): add new dmsquash-live-autooverlay module

man/dracut.cmdline.7.asc

@@ -1171,6 +1171,10 @@ rd.live.overlay=/dev/sdb1:persistent-overlay.img
 rd.live.overlay=UUID=99440c1f-8daa-41bf-b965-b7240a8996f4
 --
 
+**rd.live.overlay.cowfs=**__[btrfs|ext4|xfs]__::
+Specifies the filesystem to use when formatting the overlay partition.
+The default is ext4.
+
 **rd.live.overlay.size=**__<size_MiB>__::
 Specifies a non-persistent Device-mapper overlay size in MiB.  The default is
 _32768_.

modules.d/90dmsquash-live-autooverlay/create-overlay-genrules.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# shellcheck disable=SC2154
+case "$root" in
+    live:/dev/*)
+        printf 'SYMLINK=="%s", RUN+="/sbin/initqueue --settled --onetime --unique /sbin/create-overlay %s"\n' \
+            "${root#live:/dev/}" "${root#live:}" >> /etc/udev/rules.d/95-create-overlay.rules
+        wait_for_dev -n "${root#live:}"
+        ;;
+esac

modules.d/90dmsquash-live-autooverlay/create-overlay.sh

@@ -0,0 +1,119 @@
+#!/bin/sh
+
+type getarg > /dev/null 2>&1 || . /lib/dracut-lib.sh
+
+if getargbool 0 rd.live.debug -n -y rdlivedebug; then
+    exec > /tmp/create-overlay.$$.out
+    exec 2>> /tmp/create-overlay.$$.out
+    set -x
+fi
+
+gatherData() {
+    overlay=$(getarg rd.live.overlay)
+    if [ -z "$overlay" ]; then
+        info "Skipping overlay creation: kernel command line parameter 'rd.live.overlay' is not set"
+        exit 0
+    fi
+    # shellcheck disable=SC2086
+    if ! str_starts ${overlay} LABEL=; then
+        die "Overlay creation failed: the partition must be set by LABEL in the 'rd.live.overlay' kernel parameter"
+    fi
+
+    overlayLabel=${overlay#LABEL=}
+    # shellcheck disable=SC2086
+    if [ -b /dev/disk/by-label/${overlayLabel} ]; then
+        info "Skipping overlay creation: overlay already exists"
+        exit 0
+    fi
+
+    filesystem=$(getarg rd.live.overlay.cowfs)
+    [ -z "$filesystem" ] && filesystem="ext4"
+    if [ "$filesystem" != "ext4" ] && [ "$filesystem" != "xfs" ] && [ "$filesystem" != "btrfs" ]; then
+        die "Overlay creation failed: only ext4, xfs, and btrfs are supported in the 'rd.live.overlay.cowfs' kernel parameter"
+    fi
+
+    live_dir=$(getarg rd.live.dir)
+    [ -z "$live_dir" ] && live_dir="LiveOS"
+
+    [ -z "$1" ] && exit 1
+    rootDevice=$1
+
+    # The kernel command line's 'root=' parameter was parsed into the $root variable by the dmsquash-live module.
+    # $root contains the path to a symlink within /dev/disk/by-label, which points to a partition.
+    # This script needs that partition's parent block device.
+    # shellcheck disable=SC2046
+    # shellcheck disable=SC2086
+    rootDeviceAbsolutePath=$(readlink -f ${rootDevice})
+    rootDeviceSysfsPath=/sys/class/block/${rootDeviceAbsolutePath##*/}
+    if [ -f "${rootDeviceSysfsPath}/partition" ]; then
+        # shellcheck disable=SC2086
+        partition=$(cat ${rootDeviceSysfsPath}/partition)
+    else
+        partition=0
+    fi
+    # shellcheck disable=SC2086
+    readonly=$(cat ${rootDeviceSysfsPath}/ro)
+    # shellcheck disable=SC2086
+    if [ "$partition" != "1" ] || [ "$readonly" != "0" ]; then
+        info "Skipping overlay creation: unpartitioned or read-only media detected"
+        exit 0
+    fi
+    # shellcheck disable=SC2046
+    # shellcheck disable=SC2086
+    fullDriveSysfsPath=$(readlink -f ${rootDeviceSysfsPath}/..)
+    blockDevice=/dev/${fullDriveSysfsPath##*/}
+    currentPartitionCount=$(grep --count -E "${blockDevice#/dev/}[0-9]+" /proc/partitions)
+
+    # shellcheck disable=SC2086
+    freeSpaceStart=$(parted --script ${blockDevice} unit % print free \
+        | awk -v x=${currentPartitionCount} '$1 == x {getline; print $1}')
+    if [ -z "$freeSpaceStart" ]; then
+        info "Skipping overlay creation: there is no free space after the last partition"
+        exit 0
+    fi
+    partitionStart=$((${freeSpaceStart%.*} + 1))
+    if [ $partitionStart -eq 100 ]; then
+        info "Skipping overlay creation: there is not enough free space after the last partition"
+        exit 0
+    fi
+
+    overlayPartition=${blockDevice}$((currentPartitionCount + 1))
+
+    label=$(blkid --match-tag LABEL --output value "$rootDevice")
+    uuid=$(blkid --match-tag UUID --output value "$rootDevice")
+    if [ -z "$label" ] || [ -z "$uuid" ]; then
+        die "Overlay creation failed: failed to look up root device label and UUID"
+    fi
+}
+
+createPartition() {
+    # shellcheck disable=SC2086
+    parted --script --align optimal ${blockDevice} mkpart primary ${partitionStart}% 100%
+}
+
+createFilesystem() {
+    # shellcheck disable=SC2086
+    mkfs.${filesystem} -L ${overlayLabel} ${overlayPartition}
+
+    baseDir=/run/initramfs/create-overlayfs
+    mkdir -p ${baseDir}
+    # shellcheck disable=SC2086
+    mount -t auto ${overlayPartition} ${baseDir}
+
+    mkdir -p ${baseDir}/${live_dir}/ovlwork
+    # shellcheck disable=SC2086
+    mkdir ${baseDir}/${live_dir}/overlay-${label}-${uuid}
+
+    umount ${baseDir}
+    rm -r ${baseDir}
+}
+
+main() {
+    gatherData "$1"
+    createPartition
+    udevsettle
+    createFilesystem
+    udevsettle
+}
+
+main "$1"

modules.d/90dmsquash-live-autooverlay/module-setup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+check() {
+    # including a module dedicated to live environments in a host-only initrd doesn't make sense
+    [[ $hostonly ]] && return 1
+    return 255
+}
+
+depends() {
+    echo dmsquash-live
+    return 0
+}
+
+installkernel() {
+    instmods btrfs ext4 xfs
+}
+
+install() {
+    inst_multiple awk blkid cat grep mkdir mount parted readlink rmdir tr umount
+    inst_multiple -o mkfs.btrfs mkfs.ext4 mkfs.xfs
+    # shellcheck disable=SC2154
+    inst_hook pre-udev 25 "$moddir/create-overlay-genrules.sh"
+    inst_script "$moddir/create-overlay.sh" "/sbin/create-overlay"
+    dracut_need_initqueue
+}

pkgbuild/dracut.spec

@@ -142,7 +142,7 @@ Requires: %{name} >= %{version}-%{dist_free_release}
 Requires: %{name} = %{version}-%{release}
 %endif
 Requires: %{name}-network = %{version}-%{release}
-Requires: tar gzip coreutils bash device-mapper curl
+Requires: tar gzip coreutils bash device-mapper curl parted
 %if 0%{?fedora}
 Requires: fuse ntfs-3g
 %endif
@@ -466,6 +466,7 @@ echo 'dracut_rescue_image="yes"' > $RPM_BUILD_ROOT%{dracutlibdir}/dracut.conf.d/
 %files live
 %{dracutlibdir}/modules.d/99img-lib
 %{dracutlibdir}/modules.d/90dmsquash-live
+%{dracutlibdir}/modules.d/90dmsquash-live-autooverlay
 %{dracutlibdir}/modules.d/90dmsquash-live-ntfs
 %{dracutlibdir}/modules.d/90livenet
 

test/TEST-16-DMSQUASH/create-root.sh

@@ -11,9 +11,17 @@ udevadm control --reload
 set -e
 
 udevadm settle
-mkfs.ext4 -q -L dracut /dev/disk/by-id/ata-disk_root
+
+# create a single partition using 50% of the capacity of the image file created by test_setup() in test.sh
+sfdisk /dev/disk/by-id/ata-disk_root << EOF
+2048,161792
+EOF
+
+udevadm settle
+
+mkfs.ext4 -q -L dracut /dev/disk/by-id/ata-disk_root-part1
 mkdir -p /root
-mount /dev/disk/by-id/ata-disk_root /root
+mount /dev/disk/by-id/ata-disk_root-part1 /root
 mkdir -p /root/run /root/testdir
 cp -a -t /root /source/*
 echo "Creating squashfs"

test/TEST-16-DMSQUASH/test-init.sh

@@ -9,6 +9,12 @@ exec > /dev/console 2>&1
 
 echo "dracut-root-block-success" | dd oflag=direct,dsync of=/dev/disk/by-id/ata-disk_marker
 
+if grep -qF ' rd.live.overlay=LABEL=persist ' /proc/cmdline; then
+    # Writing to a file in the root filesystem lets test_run() verify that the autooverlay module successfully created
+    # and formatted the overlay partition and that the dmsquash-live module used it when setting up the rootfs overlay.
+    echo "dracut-autooverlay-success" > /overlay-marker
+fi
+
 export TERM=linux
 export PS1='initramfs-test:\w\$ '
 [ -f /etc/mtab ] || ln -sfn /proc/mounts /etc/mtab

test/TEST-16-DMSQUASH/test.sh

@@ -5,8 +5,9 @@ TEST_DESCRIPTION="live root on a squash filesystem"
 
 KVERSION="${KVERSION-$(uname -r)}"
 
-# Uncomment this to debug failures
-#DEBUGFAIL="rd.shell rd.debug loglevel=7"
+# Uncomment these to debug failures
+#DEBUGFAIL="rd.shell rd.debug rd.live.debug loglevel=7"
+#DEBUGTOOLS="setsid ls cat sfdisk"
 
 test_run() {
     dd if=/dev/zero of="$TESTDIR"/marker.img bs=1MiB count=1
@@ -29,6 +30,27 @@ test_run() {
         -initrd "$TESTDIR"/initramfs.testing
 
     grep -U --binary-files=binary -F -m 1 -q dracut-root-block-success -- "$TESTDIR"/marker.img || return 1
+
+    rootPartitions=$(sfdisk -d "$TESTDIR"/root.img | grep -c 'root\.img[0-9]')
+    [ "$rootPartitions" -eq 1 ] || return 1
+
+    "$testdir"/run-qemu \
+        "${disk_args[@]}" \
+        -boot order=d \
+        -append "rd.live.image rd.live.overlay.overlayfs=1 rd.live.overlay=LABEL=persist rd.live.dir=testdir root=LABEL=dracut console=ttyS0,115200n81 quiet selinux=0 rd.info rd.shell=0 panic=1 oops=panic softlockup_panic=1 $DEBUGFAIL" \
+        -initrd "$TESTDIR"/initramfs.testing-autooverlay
+
+    rootPartitions=$(sfdisk -d "$TESTDIR"/root.img | grep -c 'root\.img[0-9]')
+    [ "$rootPartitions" -eq 2 ] || return 1
+
+    (
+        # Ensure that this test works when run with the `V=1` parameter, which runs the script with `set -o pipefail`.
+        set +o pipefail
+
+        # Verify that the string "dracut-autooverlay-success" occurs in the second partition in the image file.
+        dd if="$TESTDIR"/root.img bs=1MiB skip=80 status=none \
+            | grep -U --binary-files=binary -F -m 1 -q dracut-autooverlay-success
+    ) || return 1
 }
 
 test_setup() {
@@ -55,7 +77,7 @@ test_setup() {
         ln -s dracut-util "${initdir}/usr/bin/dracut-getarg"
         ln -s dracut-util "${initdir}/usr/bin/dracut-getargs"
 
-        inst_multiple mkdir ln dd stty mount poweroff
+        inst_multiple mkdir ln dd stty mount poweroff grep "$DEBUGTOOLS"
 
         cp -a -- /etc/ld.so.conf* "$initdir"/etc
         ldconfig -r "$initdir"
@@ -121,6 +143,16 @@ test_setup() {
         --force "$TESTDIR"/initramfs.testing "$KVERSION" || return 1
 
     ls -sh "$TESTDIR"/initramfs.testing
+
+    "$basedir"/dracut.sh -l -i "$TESTDIR"/overlay / \
+        --modules "dmsquash-live-autooverlay qemu" \
+        --omit "rngd" \
+        --drivers "ext4 sd_mod" \
+        --no-hostonly --no-hostonly-cmdline \
+        --force "$TESTDIR"/initramfs.testing-autooverlay "$KVERSION" || return 1
+
+    ls -sh "$TESTDIR"/initramfs.testing-autooverlay
+
     rm -rf -- "$TESTDIR"/overlay
 }
 

test/container/Dockerfile-Arch

@@ -12,7 +12,7 @@ RUN pacman --noconfirm -Sy \
     linux dash strace dhclient asciidoc cpio pigz squashfs-tools \
     qemu btrfs-progs mdadm dmraid nfs-utils nfsidmap lvm2 nbd \
     dhcp networkmanager multipath-tools vi tcpdump open-iscsi \
-    git shfmt shellcheck astyle which base-devel glibc && yes | pacman -Scc
+    git shfmt shellcheck astyle which base-devel glibc parted && yes | pacman -Scc
 
 RUN useradd -m build
 RUN su build -c 'cd && git clone https://aur.archlinux.org/perl-config-general.git && cd perl-config-general && makepkg -s --noconfirm'

test/container/Dockerfile-Debian

@@ -44,6 +44,7 @@ RUN apt-get update -y -qq && apt-get upgrade -y -qq && DEBIAN_FRONTEND=nonintera
     network-manager \
     nfs-kernel-server \
     open-iscsi \
+    parted \
     pigz \
     pkg-config \
     procps \

test/container/Dockerfile-Fedora-latest

@@ -49,6 +49,7 @@ RUN dnf -y install --setopt=install_weak_deps=False \
     which \
     ShellCheck \
     shfmt \
+    parted \
     && dnf -y update && dnf clean all
 
 # Set default command

test/container/Dockerfile-Gentoo

@@ -17,13 +17,18 @@ LABEL RUN="docker run -it --name NAME --privileged --ipc=host --net=host --pid=h
 
 RUN echo 'export DRACUT_NO_XATTR=1 KVERSION=$(cd /lib/modules; ls -1 | tail -1)' > /etc/profile.d/dracut-test.sh
 
+# Only install `dmsetup`: attempting to install all of lvm2 fails due to missing kernel headers.
+RUN echo 'sys-fs/lvm2 device-mapper-only -thin' > /etc/portage/package.use/lvm2
+
 # Install needed packages for the dracut CI container
 RUN emerge -qv \
     app-arch/cpio \
     app-emulation/qemu \
     app-shells/dash \
     sys-apps/busybox \
+    sys-block/parted \
     sys-fs/btrfs-progs \
+    sys-fs/lvm2 \
     sys-fs/squashfs-tools \
     && rm -rf /var/cache/* /usr/share/doc/* /usr/share/man/* /var/db/repos/gentoo
 

test/container/Dockerfile-OpenSuse-latest

@@ -13,7 +13,7 @@ RUN dnf -y install --setopt=install_weak_deps=False \
     strace libkmod-devel gcc bzip2 xz tar wget rpm-build make git bash-completion \
     sudo kernel dhcp-client qemu-kvm /usr/bin/qemu-system-$(uname -m) e2fsprogs \
     tcpdump iproute iputils kbd NetworkManager btrfsprogs tgt dbus-broker \
-    iscsiuio open-iscsi which ShellCheck procps pigz \
+    iscsiuio open-iscsi which ShellCheck procps pigz parted squashfs \
     && dnf -y update && dnf clean all
 
 RUN shfmt_version=3.2.4; wget "https://github.com/mvdan/sh/releases/download/v${shfmt_version}/shfmt_v${shfmt_version}_linux_amd64" -O /usr/local/bin/shfmt \