Add support for osmet packing

This is the server side required for osmet packing:
coreos/coreos-installer#187

We run `osmet pack` when creating the live ISO and include the osmet
binary as part of the CPIO containing the root squashfs.

One thing to note is that the metal image is now always required before
building the live ISO. This is because the whole point of osmet is to
efficiently pack the metal image.

Since osmet obsoletes the `fulliso` artifact, this PR also removes
support for it.

src/cmd-buildextend-installer

@@ -43,6 +43,9 @@ parser = argparse.ArgumentParser()
 parser.add_argument("--build", help="Build ID")
 parser.add_argument("--force", action='store_true', default=False,
                     help="Overwrite previously generated installer")
+# Temporary switch while we ratchet osmet in place
+parser.add_argument("--osmet", action='store_true', default=False,
+                    help="Generate osmet file")
 args = parser.parse_args()
 
 # Identify the builds and target the latest build if none provided
@@ -57,19 +60,13 @@ squashfs_compression = image_yaml.get('squashfs-compression', 'zstd')
 
 # Hacky mode switch, until we can drop support for the installer images
 is_live = os.path.basename(sys.argv[0]).endswith('-live')
-is_fulliso = os.path.basename(sys.argv[0]).endswith('-fulliso')
-if is_fulliso:
-    is_live = True
-    image_type = 'fulliso'
-elif is_live:
+if is_live:
     image_type = 'live'
 else:
     image_type = 'installer'
 meta_keys = {k: 'live-' + k if is_live else k for k in ('iso', 'kernel', 'initramfs')}
 
 config_src = image_type
-if image_type == 'fulliso':
-    config_src = 'live'
 srcdir_prefix = f"src/config/{config_src}/"
 
 if not os.path.isdir(srcdir_prefix):
@@ -142,12 +139,11 @@ def generate_iso():
 
     tmpisofile = os.path.join(tmpdir, iso_name)
     img_metal_obj = buildmeta['images'].get('metal')
-    img_metal = None
-    if img_metal_obj is not None:
-        img_metal = os.path.join(builddir, img_metal_obj['path'])
-        img_src = img_metal
-    else:
-        img_src = os.path.join(builddir, buildmeta['images']['qemu']['path'])
+    if img_metal_obj is None:
+        raise Exception("ISO generation requires `metal` image")
+
+    img_metal = os.path.join(builddir, img_metal_obj['path'])
+    img_metal_checksum = img_metal_obj['sha256']
 
     # Find the directory under `/usr/lib/modules/<kver>` where the
     # kernel/initrd live. It will be the 2nd entity output by
@@ -174,42 +170,26 @@ def generate_iso():
     elif image_type == 'installer':
         initramfs_cpio_ext = os.path.join(tmpdir, 'legacy-stamp-initramfs')
         generate_initramfs_stampfile(tmpdir, initramfs_cpio_ext, 'etc/coreos-legacy-installer-initramfs')
-    if is_fulliso:
-        tmp_initramfs = os.path.join(tmpdir, 'initramfs')
-
-        if img_metal is None:
-            raise Exception("fulliso requires `metal` image")
-        metal_dest_basename = os.path.basename(img_metal) + '.xz'
-        metal_dest = os.path.join(tmpisoroot, metal_dest_basename)
-        with open(metal_dest, 'wb') as f:
-            run_verbose(['xz', '-c9', '-T2', img_metal], stdout=f)
-        os.symlink(metal_dest_basename, os.path.join(tmpisoroot, 'image-metal.xz'))
-
-        # In the fulliso case, we copy the squashfs to the ISO root.
-        print(f'Compressing squashfs with {squashfs_compression}')
-        run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
-                    img_src, os.path.join(tmpisoroot, 'root.squashfs'), squashfs_compression])
-        # Just pad with NUL bytes for the ISO image.  We'll truncate the
-        # padding off again when copying the PXE initrd.
-        with open(tmp_initramfs, 'wb') as fdst:
-            with open(initramfs, 'rb') as fsrc:
-                shutil.copyfileobj(fsrc, fdst)
-            with open(initramfs_cpio_ext, 'rb') as fsrc:
-                shutil.copyfileobj(fsrc, fdst)
-            fdst.write(bytes(initrd_ignition_padding))
-        os.rename(tmp_initramfs, initramfs)
-    elif is_live:
+    if is_live:
         tmp_squashfs = os.path.join(tmpdir, 'root.squashfs')
         tmp_cpio = os.path.join(tmpdir, 'root.cpio')
         tmp_initramfs = os.path.join(tmpdir, 'initramfs')
+        cpio_input_files = [os.path.basename(tmp_squashfs)]
+        if args.osmet:
+            tmp_osmet = os.path.join(tmpdir, img_metal_obj['path'] + '.osmet')
+            cpio_input_files += [os.path.basename(tmp_osmet)]
+            print(f'Generating osmet file')
+            run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
+                         img_metal, tmp_osmet, img_metal_checksum])
 
         print(f'Compressing squashfs with {squashfs_compression}')
         run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
-                     img_src, tmp_squashfs, squashfs_compression])
+                     img_metal, tmp_squashfs, squashfs_compression])
+        # create a CPIO layer which holds the root squashfs and osmet binary
         run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
                 '--quiet', '--reproducible', '--force-local',
                 '-D', os.path.dirname(tmp_squashfs), '-O', tmp_cpio],
-                input=os.path.basename(tmp_squashfs).encode())
+                input=('\n'.join(cpio_input_files).encode()))
         # Compression is redundant but the kernel requires it
         run_verbose(['gzip', '-1', tmp_cpio])
 
@@ -238,7 +218,7 @@ def generate_iso():
 
     # Read and filter kernel arguments for substituting into ISO bootloader
     result = run_verbose(['/usr/lib/coreos-assembler/gf-get-kargs',
-            img_src], stdout=subprocess.PIPE, text=True)
+            img_metal], stdout=subprocess.PIPE, text=True)
     kargs_array = [karg for karg in result.stdout.split()
             if karg.split('=')[0] not in live_exclude_kargs]
     kargs_array.append(f"coreos.liveiso={volid}")

src/osmet-pack

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -euo pipefail
+
+if [ ! -f /etc/cosa-supermin ]; then
+    dn=$(dirname "$0")
+    # shellcheck source=src/cmdlib.sh
+    . "${dn}"/cmdlib.sh
+
+    img_src=$1; shift
+    osmet_dest=$1; shift
+    checksum=$1; shift
+    coreinst=${1:-${OSMET_PACK_COREOS_INSTALLER:-}}
+
+    workdir=$(pwd)
+    TMPDIR=tmp/tmp-osmet-pack
+    rm -rf "${TMPDIR}"
+    mkdir -p "${TMPDIR}"
+
+    if [[ $img_src == *.gz || $img_src == *.xz ]]; then
+        img="$(basename "$img_src")"
+        fatal "Cannot pack osmet from $img; not an uncompressed image"
+    fi
+
+    set -- "${TMPDIR}/osmet.bin" "${checksum}"
+    if [ -n "${coreinst:-}" ]; then
+        cp "${coreinst}" "${TMPDIR}/coreos-installer"
+        set -- "$@" "${TMPDIR}/coreos-installer"
+    fi
+
+    # stamp it with "osmet" serial so we find it easily in the VM
+    runvm -drive "if=none,id=osmet,format=raw,readonly=on,file=${img_src}" \
+        -device virtio-blk,serial=osmet,drive=osmet -- \
+        /usr/lib/coreos-assembler/osmet-pack "$@"
+
+    mv "${TMPDIR}/osmet.bin" "${osmet_dest}"
+    rm -rf "${TMPDIR}"
+
+    exit 0
+fi
+
+# This runs inside supermin
+
+osmet_dest=$1; shift
+checksum=$1; shift
+coreinst=${1:-}
+
+set -x
+
+# we want /dev/disk symlinks for coreos-installer
+/usr/lib/systemd/systemd-udevd --daemon
+/usr/sbin/udevadm trigger --settle
+
+rootfs=/dev/disk/by-id/virtio-osmet-part4
+
+mkdir -p /sysroot
+mount -o ro "${rootfs}" /sysroot
+osname=$(ls /sysroot/ostree/deploy)
+deploydir=$(find "/sysroot/ostree/deploy/$osname/deploy" -mindepth 1 -maxdepth 1 -type d)
+# shellcheck disable=SC1090
+description=$(. "${deploydir}/etc/os-release" && echo "${PRETTY_NAME}")
+
+if [ -z "${coreinst}" ]; then
+    coreinst=${deploydir}/usr/bin/coreos-installer
+fi
+
+RUST_BACKTRACE=full "${coreinst}" osmet pack /dev/disk/by-id/virtio-osmet \
+    --description "${description}" \
+    --checksum "${checksum}" \
+    --output "${osmet_dest}"