Add support for osmet packing

This is the server side required for osmet packing:
coreos/coreos-installer#187

We run `osmet pack` when creating the live ISO and include the osmet
binary as part of the CPIO containing the root squashfs.

src/cmd-buildextend-installer

@@ -57,19 +57,13 @@ squashfs_compression = image_yaml.get('squashfs-compression', 'zstd')
 
 # Hacky mode switch, until we can drop support for the installer images
 is_live = os.path.basename(sys.argv[0]).endswith('-live')
-is_fulliso = os.path.basename(sys.argv[0]).endswith('-fulliso')
-if is_fulliso:
-    is_live = True
-    image_type = 'fulliso'
-elif is_live:
+if is_live:
     image_type = 'live'
 else:
     image_type = 'installer'
 meta_keys = {k: 'live-' + k if is_live else k for k in ('iso', 'kernel', 'initramfs')}
 
 config_src = image_type
-if image_type == 'fulliso':
-    config_src = 'live'
 srcdir_prefix = f"src/config/{config_src}/"
 
 if not os.path.isdir(srcdir_prefix):
@@ -126,12 +120,11 @@ def generate_iso():
 
     tmpisofile = os.path.join(tmpdir, iso_name)
     img_metal_obj = buildmeta['images'].get('metal')
-    img_metal = None
-    if img_metal_obj is not None:
-        img_metal = os.path.join(builddir, img_metal_obj['path'])
-        img_src = img_metal
-    else:
-        img_src = os.path.join(builddir, buildmeta['images']['qemu']['path'])
+    if img_metal_obj is None:
+        raise Exception("ISO generation requires `metal` image")
+
+    img_metal = os.path.join(builddir, img_metal_obj['path'])
+    img_metal_checksum = img_metal_obj['sha256']
 
     # Find the directory under `/usr/lib/modules/<kver>` where the
     # kernel/initrd live. It will be the 2nd entity output by
@@ -158,50 +151,33 @@ def generate_iso():
             live_initramfs_stamp_path = 'etc/coreos-live-initramfs'
             live_initramfs_stamp = os.path.join(livetmpd, live_initramfs_stamp_path)
             os.makedirs(os.path.dirname(live_initramfs_stamp), exist_ok=True)
-            with open(live_initramfs_stamp, 'w') as f:
+            with open(live_initramfs_stamp, 'w'):
                 pass
             run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
                 '--quiet', '--reproducible', '--force-local',
                 '-D', livetmpd, '-O', live_initramfs_cpio],
                 input=live_initramfs_stamp_path.encode())
             run_verbose(['gzip', live_initramfs_cpio])
             live_initramfs_cpio = live_initramfs_cpio + '.gz'
-    if is_fulliso:
-        tmp_initramfs = os.path.join(tmpdir, 'initramfs')
 
-        if img_metal is None:
-            raise Exception("fulliso requires `metal` image")
-        metal_dest_basename = os.path.basename(img_metal) + '.xz'
-        metal_dest = os.path.join(tmpisoroot, metal_dest_basename)
-        with open(metal_dest, 'wb') as f:
-            run_verbose(['xz', '-c9', '-T2', img_metal], stdout=f)
-        os.symlink(metal_dest_basename, os.path.join(tmpisoroot, 'image-metal.xz'))
-
-        # In the fulliso case, we copy the squashfs to the ISO root.
-        print(f'Compressing squashfs with {squashfs_compression}')
-        run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
-                    img_src, os.path.join(tmpisoroot, 'root.squashfs'), squashfs_compression])
-        # Just pad with NUL bytes for the ISO image.  We'll truncate the
-        # padding off again when copying the PXE initrd.
-        with open(tmp_initramfs, 'wb') as fdst:
-            with open(initramfs, 'rb') as fsrc:
-                shutil.copyfileobj(fsrc, fdst)
-            with open(live_initramfs_cpio, 'rb') as fsrc:
-                shutil.copyfileobj(fsrc, fdst)
-            fdst.write(bytes(initrd_ignition_padding))
-        os.rename(tmp_initramfs, initramfs)
-    elif is_live:
         tmp_squashfs = os.path.join(tmpdir, 'root.squashfs')
         tmp_cpio = os.path.join(tmpdir, 'root.cpio')
+        tmp_osmet = os.path.join(tmpdir, img_metal_obj['path'] + '.osmet')
         tmp_initramfs = os.path.join(tmpdir, 'initramfs')
 
+        print(f'Generating osmet file')
+        run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
+                     img_metal, tmp_osmet, img_metal_checksum])
+
         print(f'Compressing squashfs with {squashfs_compression}')
         run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
-                     img_src, tmp_squashfs, squashfs_compression])
+                     img_metal, tmp_squashfs, squashfs_compression])
+        # create a CPIO layer which holds the root squashfs and osmet binary
+        cpio_input_files = [os.path.basename(f) for f in [tmp_squashfs, tmp_osmet]]
         run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
                 '--quiet', '--reproducible', '--force-local',
                 '-D', os.path.dirname(tmp_squashfs), '-O', tmp_cpio],
-                input=os.path.basename(tmp_squashfs).encode())
+                input=('\n'.join(cpio_input_files)).encode())
         # Compression is redundant but the kernel requires it
         run_verbose(['gzip', '-1', tmp_cpio])
 
@@ -221,7 +197,7 @@ def generate_iso():
 
     # Read and filter kernel arguments for substituting into ISO bootloader
     result = run_verbose(['/usr/lib/coreos-assembler/gf-get-kargs',
-            img_src], stdout=subprocess.PIPE, text=True)
+            img_metal], stdout=subprocess.PIPE, text=True)
     kargs_array = [karg for karg in result.stdout.split()
             if karg.split('=')[0] not in live_exclude_kargs]
     kargs_array.append(f"coreos.liveiso={volid}")

src/osmet-pack

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -euo pipefail
+
+if [ ! -f /etc/cosa-supermin ]; then
+    dn=$(dirname "$0")
+    # shellcheck source=src/cmdlib.sh
+    . "${dn}"/cmdlib.sh
+
+    img_src=$1; shift
+    osmet_dest=$1; shift
+    checksum=$1; shift
+    coreinst=${1:-${OSMET_PACK_COREOS_INSTALLER:-}}
+
+    workdir=$(pwd)
+    TMPDIR=tmp/tmp-osmet-pack
+    rm -rf "${TMPDIR}"
+    mkdir -p "${TMPDIR}"
+
+    if [[ $img_src == *.gz || $img_src == *.xz ]]; then
+        img="$(basename "$img_src")"
+        fatal "Cannot pack osmet from $img; not an uncompressed image"
+    fi
+
+    set -- "${TMPDIR}/osmet.bin" "${checksum}"
+    if [ -n "${coreinst:-}" ]; then
+        cp "${coreinst}" "${TMPDIR}/coreos-installer"
+        set -- "$@" "${TMPDIR}/coreos-installer"
+    fi
+
+    # stamp it with "osmet" serial so we find it easily in the VM
+    runvm -drive "if=none,id=osmet,format=raw,readonly=on,file=${img_src}" \
+        -device virtio-blk,serial=osmet,drive=osmet -- \
+        /usr/lib/coreos-assembler/osmet-pack "$@"
+
+    mv "${TMPDIR}/osmet.bin" "${osmet_dest}"
+    rm -rf "${TMPDIR}"
+
+    exit 0
+fi
+
+# This runs inside supermin
+
+osmet_dest=$1; shift
+checksum=$1; shift
+coreinst=${1:-}
+
+set -x
+
+# we want /dev/disk symlinks for coreos-installer
+/usr/lib/systemd/systemd-udevd --daemon
+/usr/sbin/udevadm trigger --settle
+
+rootfs=/dev/disk/by-id/virtio-osmet-part4
+
+mkdir -p /sysroot
+mount -o ro "${rootfs}" /sysroot
+osname=$(ls /sysroot/ostree/deploy)
+deploydir=$(find "/sysroot/ostree/deploy/$osname/deploy" -mindepth 1 -maxdepth 1 -type d)
+# shellcheck disable=SC1090
+description=$(. "${deploydir}/etc/os-release" && echo "${PRETTY_NAME}")
+
+if [ -z "${coreinst}" ]; then
+    coreinst=${deploydir}/usr/bin/coreos-installer
+fi
+
+RUST_BACKTRACE=full "${coreinst}" osmet pack /dev/disk/by-id/virtio-osmet \
+    --description "${description}" \
+    --checksum "${checksum}" \
+    --output "${osmet_dest}"