Use supermin in unprivileged environments

This is a rebased rework of #124 with some modifications:
- We auto-detect if we have CAP_SYS_ADMIN and if not, fall back to using
  supermin. My position is that both approaches will be in use in CI
  contexts and that the privileged case is faster for local dev, where
  iterating fast on the content will matter. I've also hopefully
  implemented things in a way that maintains almost the exact same logic
  build-wise between the two flows so there's not too much divergence.
  Anyway, totally open to revisiting this if needed!
- In the virtualized path, `fetch` now directly populates the qcow2
  cache so that the split `fetch`/`build` approach keeps working as
  expected.
- We drop the repo-build/ repo since it's essentially also a cache and
  duplicates content from the archive repo. This is also needed to
  ensure that the pkgcache repo and the repo we commit into are both on
  the same file system.
- The supermin appliance is reused if already generated; the `runvm`
  command just takes the command you want to run verbatim and plops it
  into a file the appliance is already coded to check from.

Some other minor fixes:
- We handle symlinked repos.
- Split out supermin packages into a separate file.
- Capture rc and bubble that up to the `runvm` caller.
- Add virtio-rng device.

Originally based on a patch by:
Dusty Mabe <[email protected]>

Dockerfile

@@ -2,7 +2,7 @@ FROM registry.fedoraproject.org/fedora:28
 WORKDIR /root/containerbuild
 
 # Only need a few of our scripts for the first few steps
-COPY ./build.sh ./deps.txt ./build-deps.txt /root/containerbuild/
+COPY ./build.sh ./deps.txt ./vmdeps.txt ./build-deps.txt /root/containerbuild/
 RUN ./build.sh configure_yum_repos
 RUN ./build.sh install_rpms
 

src/cmd-build

@@ -145,12 +145,12 @@ else
 fi
 
 if [ -n "${previous_build}" ]; then
-    rpm-ostree --repo=${workdir}/repo-build db diff ${previous_commit} ${commit}
+    rpm-ostree --repo=${workdir}/repo db diff ${previous_commit} ${commit}
 fi
 
 image_input_checksum=$((echo ${commit} && echo ${kickstart_checksum}) | sha256sum_str)
 echo "New image input checksum: ${image_input_checksum}"
-version=$(ostree --repo=${workdir}/repo-build show --print-metadata-key=version ${commit} | sed -e "s,',,g")
+version=$(ostree --repo=${workdir}/repo show --print-metadata-key=version ${commit} | sed -e "s,',,g")
 if [ "${previous_commit}" = "${commit}" ]; then
     image_genver=$((${previous_image_genver} + 1))
     buildid=${version}-${image_genver}
@@ -160,15 +160,6 @@ else
 fi
 echo "New build ID: ${buildid}"
 
-# https://github.com/ostreedev/ostree/issues/1562#issuecomment-385393872
-# The passwd files (among others) don't have world readability.  This won't
-# actually corrupt the repository as the *canonical* permissions are stored
-# as xattrs.  Probably what we should do is have an ostree option to specify
-# a permission mask for objects.
-sudo chmod -R a+rX ${workdir}/repo-build/objects
-ostree --repo=${workdir}/repo pull-local ${workdir}/repo-build "${ref:-${commit}}"
-ostree --repo=${workdir}/repo summary -u
-
 # Generate JSON
 if [ -n "${previous_commit}" ]; then
     previous_commit_json='"'"${previous_commit}"'"'

src/cmd-clean

@@ -49,8 +49,7 @@ prepare_build
 
 # But go back to the toplevel
 cd ${workdir}
-# Note we don't prune the cache/ dir or the objects
-# in the repos.  If you want that, just rm -rf them.
+# Note we don't prune the cache.qcow2 or the objects
+# in the repo.  If you want that, just rm -rf them.
 rm -rf repo/refs/heads/* builds/* tmp/*
 ostree --repo=repo summary -u
-sudo rm -rf repo-build/refs/heads/*

src/cmd-init

@@ -116,4 +116,7 @@ mkdir -p cache
 mkdir -p builds
 mkdir -p tmp
 ostree --repo=repo init --mode=archive
-ostree --repo=repo-build init --mode=bare-user
+if ! has_privileges; then
+    LIBGUESTFS_BACKEND=direct qemu-img create -f qcow2 cache.qcow2 10G
+    virt-format --filesystem=xfs -a cache.qcow2
+fi

src/cmdlib.sh

@@ -1,7 +1,30 @@
 # Shared shell script library
 
+info() {
+    echo "info: $@" 1>&2
+}
+
 fatal() {
-    echo "error: $@" 1>&2; exit 1
+    info "$@"; exit 1
+}
+
+_privileged=
+has_privileges() {
+    if [ -z "${_privileged:-}" ]; then
+        if [ -n "${FORCE_UNPRIVILEGED:-}" ]; then
+            info "Detected FORCE_UNPRIVILEGED; using virt"
+            _privileged=0
+        elif ! capsh --print | grep -q 'Current.*cap_sys_admin'; then
+            info "Missing CAP_SYS_ADMIN; using virt"
+            _privileged=0
+        elif ! sudo true; then
+            info "Missing sudo privs; using virt"
+            _privileged=0
+        else
+            _privileged=1
+        fi
+    fi
+    [ ${_privileged} == 1 ]
 }
 
 preflight() {
@@ -28,14 +51,6 @@ preflight() {
         fatal "Unable to find /dev/kvm"
     fi
 
-    if ! capsh --print | grep -q 'Current.*cap_sys_admin'; then
-        fatal "This container must currently be run with --privileged"
-    fi
-
-    if ! sudo true; then
-        fatal "The user must currently have sudo privileges"
-    fi
-
     # permissions on /dev/kvm vary by (host) distro.  If it's
     # not writable, recreate it.
     if ! [ -w /dev/kvm ]; then
@@ -49,6 +64,8 @@ prepare_build() {
     preflight
     if ! [ -d repo ]; then
         fatal "No $(pwd)/repo found; did you run coreos-assembler init?"
+    elif ! has_privileges && [ ! -f cache.qcow2 ]; then
+        fatal "No cache.qcow2 found; did you run coreos-assembler init?"
     fi
 
     export workdir=$(pwd)
@@ -127,10 +144,105 @@ EOF
     fi
 
     rm -f ${changed_stamp}
-    set -x
-    sudo rpm-ostree compose tree --repo=${workdir}/repo-build --cachedir=${workdir}/cache \
-         --touch-if-changed "${changed_stamp}" \
-         ${treecompose_args} \
-         ${TREECOMPOSE_FLAGS:-} ${manifest} "$@"
-    set +x
+
+    # really, we don't need to do this whole jazz for `fetch`, but meh, it's
+    # just some minimal I/O in that case
+    cat > ${TMPDIR}/compose.sh <<EOF
+SUDO=
+if [ \$(id -u) != 0 ]; then
+    SUDO=sudo
+fi
+
+# build in a temp bare-user repo in cache/ so we can hardlink from pkgcache
+\$SUDO ostree --repo=${workdir}/cache/repo-build init --mode=bare-user
+
+# seed with commit metadata of last build
+# TODO: don't rely on ${ref}
+if ostree --repo=${workdir}/repo rev-parse ${ref} 2>/dev/null; then
+    \$SUDO ostree --repo=${workdir}/cache/repo-build pull-local \
+        ${workdir}/repo ${ref} --commit-metadata-only
+fi
+
+\$SUDO rpm-ostree compose tree --repo=${workdir}/cache/repo-build \
+    --cachedir=${workdir}/cache --touch-if-changed "${changed_stamp}" \
+    ${treecompose_args} ${TREECOMPOSE_FLAGS:-} \
+    ${manifest} $@
+
+\$SUDO chmod -R a+rX ${workdir}/cache/repo-build/objects
+# can't use ${changed_stamp}; it has different semantics with --download-only
+if ostree --repo=${workdir}/cache/repo-build rev-parse ${ref} 2>/dev/null; then
+    ostree --repo=${workdir}/repo pull-local ${workdir}/cache/repo-build ${ref}
+fi
+
+# just nuke it instead of pruning, it's fast to rebuild from pkgcache hardlinks
+\$SUDO rm -rf ${workdir}/repo-build
+EOF
+
+    # this is the heart of the privs vs no privs dual path
+    if has_privileges; then
+        . ${TMPDIR}/compose.sh
+    else
+        runvm bash ${TMPDIR}/compose.sh
+    fi
+}
+
+runvm() {
+    local vmpreparedir=${workdir}/tmp/supermin.prepare
+    local vmbuilddir=${workdir}/tmp/supermin.build
+
+    # use REBUILDVM=1 if e.g. hacking on rpm-ostree/ostree and wanting to get
+    # the new bits in the VM
+    if [ ! -f ${vmbuilddir}/.done ] || [ -n "${REBUILDVM:-}" ]; then
+        rm -rf ${vmpreparedir} ${vmbuilddir}
+        mkdir -p ${vmpreparedir} ${vmbuilddir}
+
+        local rpms=
+        # then add all the base deps
+        for dep in $(grep -v '^#' /usr/lib/coreos-assembler/vmdeps.txt); do
+            rpms+="$dep "
+        done
+
+        supermin --prepare --use-installed $rpms -o "${vmpreparedir}"
+        cat > "${vmpreparedir}/init" <<EOF
+#!/bin/bash
+set -xeuo pipefail
+workdir=${workdir}
+$(cat /usr/lib/coreos-assembler/supermin-init-prelude.sh)
+rc=0
+sh ${TMPDIR}/cmd.sh || rc=\$?
+echo $rc > ${workdir}/tmp/rc
+/sbin/reboot -f
+EOF
+        chmod a+x ${vmpreparedir}/init
+        (cd ${vmpreparedir} && tar -czf init.tar.gz --remove-files init)
+        supermin --build "${vmpreparedir}" --size 5G -f ext2 -o "${vmbuilddir}"
+        touch "${vmbuilddir}/.done"
+    fi
+
+    echo "$@" > ${TMPDIR}/cmd.sh
+
+    # support local dev cases wher src/config is a symlink
+    srcvirtfs=
+    if [ -L "${workdir}/src/config" ]; then
+        # qemu follows symlinks
+        srcvirtfs="-virtfs local,id=source,path=${workdir}/src/config,security_model=none,mount_tag=source"
+    fi
+
+    qemu-kvm -nodefaults -nographic -m 2048 -no-reboot \
+        -kernel "${vmbuilddir}/kernel" \
+        -initrd "${vmbuilddir}/initrd" \
+        -netdev user,id=eth0,hostname=supermin \
+        -device virtio-net-pci,netdev=eth0 \
+        -device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 \
+        -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${vmbuilddir}/root" \
+        -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \
+        -drive if=none,id=drive-scsi0-0-0-1,discard=unmap,file="${workdir}/cache.qcow2" \
+        -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1 \
+        -virtfs local,id=workdir,path="${workdir}",security_model=none,mount_tag=workdir \
+        ${srcvirtfs} -serial stdio -append "root=/dev/sda console=ttyS0 selinux=1 enforcing=0 autorelabel=1"
+
+    if [ ! -f ${workdir}/tmp/rc ]; then
+        fatal "Couldn't find rc file, something went terribly wrong!"
+    fi
+    return $(cat ${workdir}/tmp/rc)
 }

src/deps.txt

@@ -1,3 +1,6 @@
+# For privileged ops
+supermin
+
 # We default to builder user, but sudo where necessary
 sudo
 

src/prune_builds

@@ -100,4 +100,3 @@ def ostree_prune(repo_name, sudo=False):
     subprocess.run(argv, check=True)
 
 ostree_prune('repo')
-ostree_prune('repo-build', sudo=True)

src/supermin-init-prelude.sh

@@ -0,0 +1,27 @@
+mount -t proc /proc /proc
+mount -t sysfs /sys /sys
+mount -t devtmpfs devtmpfs /dev
+
+# load selinux policy
+LANG=C /sbin/load_policy  -i
+
+# load kernel module for 9pnet_virtio for 9pfs mount
+/sbin/modprobe 9pnet_virtio
+
+# need fuse module for rofiles-fuse/bwrap during post scripts run
+/sbin/modprobe fuse
+
+# set up networking
+/usr/sbin/dhclient eth0
+
+# set up workdir
+mkdir -p ${workdir}
+mount -t 9p -o rw,trans=virtio,version=9p2000.L workdir ${workdir}
+if [ -L ${workdir}/src/config ]; then
+    mkdir -p $(readlink ${workdir}/src/config)
+    mount -t 9p -o rw,trans=virtio,version=9p2000.L source ${workdir}/src/config
+fi
+mkdir -p ${workdir}/cache
+mount /dev/sdb1 ${workdir}/cache
+
+cd ${workdir}

src/vmdeps.txt

@@ -0,0 +1,16 @@
+# Base deps for a viable VM environment.
+
+# bare essentials
+bash vim-minimal coreutils util-linux procps-ng kmod kernel-modules
+
+# for composes
+rpm-ostree distribution-gpg-keys jq
+
+# for clean reboot
+systemd
+
+# networking
+dhcp-client bind-export-libs iproute
+
+# SELinux
+selinux-policy selinux-policy-targeted policycoreutils

vmdeps.txt

@@ -0,0 +1 @@
+./src/vmdeps.txt