wip: support ignition v3

TODO: kargs needs documentation that we don't handle all cases due to
      grub.cfg eval complexity (also test this)

TODO: ensure that 99-xx-ignition-systemd-cryptsetup.rules and
      ignition-luks.conf get set up

TODO: provide an implementation for is-live-image

dracut/30ignition/99-xx-ignition-systemd-cryptsetup.rules

@@ -0,0 +1,8 @@
+SUBSYSTEM!="block", GOTO="systemd_cryptsetup_end"
+
+# This overrides systemd default behavior from 99-systemd.rules, which ignores unformatted crypto devices.
+# https://github.com/systemd/systemd/commit/90e6abaea0cfd25093aae1ad862c5c909ae55829
+# Ignition relies on unformatted crypto devices being discovered to trigger formatting
+SUBSYSTEM=="block", ENV{DM_UUID}=="CRYPT-*", ENV{ID_PART_TABLE_TYPE}=="", ENV{ID_FS_USAGE}=="", ENV{SYSTEMD_READY}="1"
+
+LABEL="systemd_cryptsetup_end"

dracut/30ignition/ignition-complete.target

@@ -0,0 +1,14 @@
+# This target is reached when Ignition finishes running. Note that it gets
+# activated *only* on first boot (or if ignition.firstboot=1 is provided).
+# Thus, it is also an API for units to use so that they are activated only on
+# first boot. Simply add a link under ignition-complete.target.requires in the
+# initrd.
+[Unit]
+Description=Ignition Complete
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+Before=initrd.target
+
+# Make sure we stop all the units before switching root
+Conflicts=initrd-switch-root.target umount.target
+Conflicts=dracut-emergency.service emergency.service emergency.target

dracut/30ignition/ignition-diskful-subsequent.target

@@ -0,0 +1,12 @@
+# This target is a combination of ignition-subsequent.target and
+# ignition-diskful.target - units here should only run when we have a
+# boot disk and *aren't* doing the first boot.
+[Unit]
+Description=Ignition Subsequent Boot Disk Setup
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+Before=ignition-subsequent.target
+
+# Make sure we stop all the units before switching root
+Conflicts=initrd-switch-root.target umount.target
+Conflicts=dracut-emergency.service emergency.service emergency.target

dracut/30ignition/ignition-diskful.target

@@ -0,0 +1,12 @@
+# This target contains Ignition units that should only run when we have a
+# boot disk, i.e. when we're not running diskless from a live image in RAM.
+# Like ignition-complete.target, it only runs on first boot.
+[Unit]
+Description=Ignition Boot Disk Setup
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+Before=ignition-complete.target
+
+# Make sure we stop all the units before switching root
+Conflicts=initrd-switch-root.target umount.target
+Conflicts=dracut-emergency.service emergency.service emergency.target

dracut/30ignition/ignition-disks.service

@@ -1,25 +1,41 @@
 [Unit]
 Description=Ignition (disks)
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
 DefaultDependencies=false
+Before=ignition-complete.target
 
+# Flatcar:
 Requires=local-fs-pre.target
 Before=local-fs-pre.target
-
 Requires=ignition-setup.service
 After=ignition-setup.service
 
-# setup networking
-Wants=systemd-networkd.service
-After=systemd-networkd.service
+# Stage order: fetch-offline [-> fetch] [-> kargs] -> disks -> mount -> files.
+After=ignition-fetch.service
+Before=ignition-mount.service
+
+# This stage runs between `basic.target` and `initrd-root-device.target`,
+# see https://www.freedesktop.org/software/systemd/man/bootup.html
+# Make sure to run before the file system checks, as sgdisk will trigger
+# udev events, potentially resulting in race conditions due to disappearing
+# devices.
 
-# generate resolv.conf
-Wants=systemd-resolved.service
-After=systemd-resolved.service
+# Note that CL runs this before `local-fs-pre.target` to allow for configs that
+# completely wipe the rootfs. Though we're not there yet. But we still run
+# before `sysroot.mount` on principle.
+Before=initrd-root-device.target
+Before=sysroot.mount
 
 OnFailure=emergency.target
 OnFailureJobMode=isolate
 
+# This stage requires udevd to detect disk partitioning changes.
+Requires=systemd-udevd.service
+After=systemd-udevd.service
+
 [Service]
 Type=oneshot
+RemainAfterExit=yes
 EnvironmentFile=/run/ignition.env
-ExecStart=/usr/bin/ignition --root=/sysroot --oem=${OEM_ID} --stage=disks
+ExecStart=/usr/bin/ignition --root=/sysroot --platform=${PLATFORM_ID} --stage=disks

dracut/30ignition/ignition-fetch-offline.service

@@ -0,0 +1,33 @@
+# This unit creates /run/ignition/neednet if networking needs to be enabled.
+# The distro is responsible for sequencing a unit between
+# ignition-fetch-offline.service and ignition-fetch.service, detecting the
+# flag file with ConditionPathExists=, and enabling networking.
+
+[Unit]
+Description=Ignition (fetch-offline)
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+DefaultDependencies=false
+Before=ignition-complete.target
+# Flatcar:
+#After=basic.target
+Wants=sockets.target paths.target slices.target
+After=sockets.target paths.target slices.target
+
+# Flatcar:
+Requires=local-fs-pre.target
+Before=local-fs-pre.target
+Requires=ignition-setup.service
+After=ignition-setup.service
+
+# Stage order: fetch-offline [-> fetch] [-> kargs] -> disks -> mount -> files.
+Before=ignition-fetch.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=/run/ignition.env
+ExecStart=/usr/bin/ignition --root=/sysroot --platform=${PLATFORM_ID} --stage=fetch-offline

dracut/30ignition/ignition-fetch.service

@@ -0,0 +1,38 @@
+[Unit]
+Description=Ignition (fetch)
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
+DefaultDependencies=false
+Before=ignition-complete.target
+# Flatcar:
+#After=basic.target
+ConditionPathExists=/run/ignition/neednet
+# Don't run if the `fetch-offline` stage successfully fetched a config
+ConditionPathExists=!/run/ignition.json
+
+# Stage order: fetch-offline [-> fetch] [-> kargs] -> disks -> mount -> files.
+# We run after the setup stage has run because it may copy in new/different
+# ignition configs for us to consume.
+After=ignition-fetch-offline.service
+Before=ignition-disks.service
+
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+# If we run, we definitely need network, so make sure we run after.
+After=network.target
+# Flatcar:
+Wants=systemd-networkd.service
+After=systemd-networkd.service
+Wants=systemd-resolved.service
+After=systemd-resolved.service
+Requires=local-fs-pre.target
+Before=local-fs-pre.target
+Requires=ignition-setup.service
+After=ignition-setup.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=/run/ignition.env
+ExecStart=/usr/bin/ignition --root=/sysroot --platform=${PLATFORM_ID} --stage=fetch

dracut/30ignition/ignition-files.service

@@ -1,30 +1,30 @@
 [Unit]
 Description=Ignition (files)
+Documentation=https://github.com/coreos/ignition
+ConditionPathExists=/etc/initrd-release
 DefaultDependencies=false
-Before=initrd-parse-etc.service
+Before=ignition-complete.target
 
+# Flatcar:
 Requires=initrd-root-fs.target
 After=initrd-root-fs.target
-
 Requires=ignition-setup.service ignition-disks.service
 After=ignition-setup.service ignition-disks.service
-
 # setup the root filesystem before we try do things like create users on it.
 Requires=initrd-setup-root.service
 After=initrd-setup-root.service
 
-# setup networking
-Wants=systemd-networkd.service
-After=systemd-networkd.service
-
-# generate resolv.conf
-Wants=systemd-resolved.service
-After=systemd-resolved.service
-
 OnFailure=emergency.target
 OnFailureJobMode=isolate
 
+# Stage order: fetch-offline [-> fetch] [-> kargs] -> disks -> mount -> files.
+After=ignition-mount.service
+
+# Run before initrd-parse-etc so that we can drop files it then picks up.
+Before=initrd-parse-etc.service
+
 [Service]
 Type=oneshot
+RemainAfterExit=yes
 EnvironmentFile=/run/ignition.env
-ExecStart=/usr/bin/ignition --root=/sysroot --oem=${OEM_ID} --stage=files
+ExecStart=/usr/bin/ignition --root=/sysroot --platform=${PLATFORM_ID} --stage=files --log-to-stdout

dracut/30ignition/ignition-generator

@@ -4,6 +4,10 @@
 
 set -e
 
+# Generators don't have logging right now
+# https://github.com/systemd/systemd/issues/15638
+exec 1>/dev/kmsg; exec 2>&1
+
 UNIT_DIR="${1:-/tmp}"
 
 cmdline=( $(</proc/cmdline) )
@@ -26,8 +30,9 @@ cmdline_bool() {
 }
 
 add_requires() {
-    local name="$1"
-    local requires_dir="${UNIT_DIR}/initrd.target.requires"
+    local name="$1"; shift
+    local target="$1"; shift
+    local requires_dir="${UNIT_DIR}/${target}.requires"
     mkdir -p "${requires_dir}"
     ln -sf "../${name}" "${requires_dir}/${name}"
 }
@@ -41,17 +46,16 @@ add_wants() {
 
 
 # This can't be done with ConditionKernelCommandLine because that always
-# starts the unit's dependencies. We want to start networkd only on first
+# starts the unit's dependencies. We want to start networking only on first
 # boot.
 if $(cmdline_bool flatcar.first_boot 0) || $(cmdline_bool coreos.first_boot 0); then
-    add_requires ignition-disks.service
-    add_requires ignition-files.service
+    add_requires ignition-complete.target initrd.target
     # Only try to mount the ESP if GRUB detected a first_boot file
     if [[ $(cmdline_arg flatcar.first_boot) = "detected" ]] || [[ $(cmdline_arg coreos.first_boot) = "detected" ]]; then
-        add_requires ignition-quench.service
+        add_requires ignition-quench.service initrd.target
     fi
     if [[ $(cmdline_arg flatcar.oem.id) == "packet" ]] || [[ $(cmdline_arg coreos.oem.id) == "packet" ]]; then
-        add_requires flatcar-static-network.service
+        add_requires flatcar-static-network.service initrd.target
     fi
 
     # On EC2, shut down systemd-networkd if ignition fails so that the instance
@@ -71,6 +75,12 @@ EOF
     if [[ $(cmdline_arg flatcar.oem.id) == "openstack" ]] || [[ $(cmdline_arg coreos.oem.id) == "openstack" ]]; then
         add_wants flatcar-openstack-hostname.service
     fi
+else
+    # If we're doing a non-Ignition (subsequent) boot, then
+    # queue a different target.  This is necessary so that units
+    # like `ignition-ostree-mount-sysroot.service`
+    # can cleanly distinguish between the two.
+    add_requires ignition-subsequent.target initrd.target
 fi
 
 # Write ignition-setup.service customized for PXE/ISO or regular boot
@@ -83,6 +93,11 @@ if [[ -z "${usr}" && -f /usr.squashfs ]]; then
     pxe=1
 else
     nopxe=1
+    if $(cmdline_bool flatcar.first_boot 0) || $(cmdline_bool coreos.first_boot 0); then
+        add_requires ignition-diskful.target ignition-complete.target
+    else
+        add_requires ignition-diskful-subsequent.target ignition-subsequent.target
+    fi
 fi
 cat > ${UNIT_DIR}/ignition-setup.service <<EOF
 [Unit]
@@ -101,17 +116,18 @@ OnFailureJobMode=isolate
 
 [Service]
 Type=oneshot
+RemainAfterExit=yes
 ExecStart=/usr/sbin/ignition-setup ${nopxe:+normal} ${pxe:+pxe}
 EOF
 
 # Call the disk UUID randomizer whenever we're not PXE booting
 # The unit will check if it needs to act or not.
 if [ "${nopxe}" = 1 ]; then
-    add_requires "disk-uuid.service"
+    add_requires "disk-uuid.service" initrd.target
 fi
 
 if [[ $(cmdline_arg flatcar.oem.id) == "digitalocean" ]] || [[ $(cmdline_arg coreos.oem.id) == "digitalocean" ]]; then
-    add_requires flatcar-digitalocean-network.service
+    add_requires flatcar-digitalocean-network.service initrd.target
 fi
 
 oem_id=pxe
@@ -124,4 +140,9 @@ if [[ "${oem_id}" = "${oem_cmdline}" ]]; then
     oem_cmdline="$(cmdline_arg coreos.oem.id ${oem_id})"
 fi
 
-echo "OEM_ID=${oem_cmdline}" > /run/ignition.env
+# Ignition changed the platform name to "aws"
+if [ "${oem_cmdline}" = "ec2" ]; then
+  oem_cmdline="aws"
+fi
+
+{ echo "OEM_ID=${oem_cmdline}" ; echo "PLATFORM_ID=${oem_cmdline}" ; } > /run/ignition.env

dracut/30ignition/ignition-kargs-helper

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Based on https://github.com/coreos/ignition/blob/02c1c638b51f5694d1d1d8305c09ff6dc3e4fa0e/examples/ignition-kargs-helper
+
+set -euxo pipefail
+
+# Handle PXE boots gracefully by not mounting any disk and instead error out early
+if [ "${OEM_ID}" = "pxe" ]; then
+  echo "error: can't set kargs for PXE boots" >&2
+  exit 1
+fi
+
+# Mount the OEM partition. Note that we mount but we don't unmount it because we
+# are run in a systemd unit with MountFlags=slave so it is unmounted for us.
+oemmnt=/mnt/oem_partition
+mkdir -p ${oemmnt}
+oemdev=/dev/disk/by-label/OEM
+mount -o rw ${oemdev} ${oemmnt}
+grubcfg="${oemmnt}/grub.cfg"
+
+# We do not handle all cases of conditional setup of linux_append or linux_console because we will not emulate the GRUB scripting logic.
+# Therefore, we only support the special lines 'set linux_append="($linux_append) ..."' which must not be starting with whitespace and should not be surrounded by an if-block.
+# Any conditional setup before or afterwards or the values of linux_console are not considered.
+orig_kernelopts="$(grep -o '^set linux_append="[^"]*' $grubcfg | sed 's,^set linux_append=",,' | sed 's,$linux_append,,' | tr '\n' ' ' | sed -e 's,^[[:space:]]*,,' -e 's,[[:space:]]*$,,')"
+# add leading and trailing whitespace to allow for easy sed replacements
+kernelopts=" $orig_kernelopts "
+
+while [[ $# -gt 0 ]]
+do
+    key="$1"
+
+    case $key in
+    --should-exist)
+        arg="$2"
+        # don't repeat the arg
+        if [[ ! "${kernelopts[*]}" =~ " ${arg} " ]]; then
+            kernelopts="$kernelopts$arg "
+        fi
+        shift 2
+        ;;
+    --should-not-exist)
+        kernelopts="$(echo "$kernelopts" | sed "s| $2 | |g")"
+        shift 2
+        ;;
+    *)
+        echo "Unknown option"
+        exit 1
+        ;;
+    esac
+done
+
+# trim the leading and trailing whitespace
+kernelopts="$(echo "$kernelopts" | sed -e 's,^[[:space:]]*,,' -e 's,[[:space:]]*$,,')"
+
+# only apply the changes & reboot if changes have been made
+if [[ "$kernelopts" != "$orig_kernelopts" ]]; then
+    # Remove all existing definitions
+    sed -i 's,^set linux_append=".*",,' $grubcfg
+    # write out a single one to replace them
+    echo "set linux_append=\"\$linux_append $kernelopts\"" >> $grubcfg
+
+    # set the first-boot flag file to make sure Ignition runs again on the next boot (e.g., when a manual first boot was forced in the GRUB menu)
+    bootmnt=/mnt/boot_partition
+    mkdir -p ${bootmnt}
+    bootdev=/dev/disk/by-label/EFI-SYSTEM
+    mount -o rw ${bootdev} ${bootmnt}
+    touch "${bootmnt}/flatcar/first_boot"
+
+    systemctl reboot --force
+fi