Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for osmet packing
Browse files Browse the repository at this point in the history
This is the server side required for osmet packing:
coreos/coreos-installer#187

We run `osmet pack` when creating the live ISO and include the osmet
binary as part of the CPIO containing the root squashfs.

One thing to note is that the metal image is now always required before
building the live ISO. This is because the whole point of osmet is to
efficiently pack the metal image.

Since osmet obsoletes the `fulliso` artifact, this PR also removes
support for it.
jlebon committed Apr 24, 2020
1 parent 3da5c18 commit 37a449e
Showing 2 changed files with 90 additions and 41 deletions.
62 changes: 21 additions & 41 deletions src/cmd-buildextend-installer
Original file line number Diff line number Diff line change
@@ -43,6 +43,9 @@ parser = argparse.ArgumentParser()
parser.add_argument("--build", help="Build ID")
parser.add_argument("--force", action='store_true', default=False,
help="Overwrite previously generated installer")
# Temporary switch while we ratchet osmet in place
parser.add_argument("--osmet", action='store_true', default=False,
help="Generate osmet file")
args = parser.parse_args()

# Identify the builds and target the latest build if none provided
@@ -57,19 +60,13 @@ squashfs_compression = image_yaml.get('squashfs-compression', 'zstd')

# Hacky mode switch, until we can drop support for the installer images
is_live = os.path.basename(sys.argv[0]).endswith('-live')
is_fulliso = os.path.basename(sys.argv[0]).endswith('-fulliso')
if is_fulliso:
is_live = True
image_type = 'fulliso'
elif is_live:
if is_live:
image_type = 'live'
else:
image_type = 'installer'
meta_keys = {k: 'live-' + k if is_live else k for k in ('iso', 'kernel', 'initramfs')}

config_src = image_type
if image_type == 'fulliso':
config_src = 'live'
srcdir_prefix = f"src/config/{config_src}/"

if not os.path.isdir(srcdir_prefix):
@@ -126,12 +123,11 @@ def generate_iso():

tmpisofile = os.path.join(tmpdir, iso_name)
img_metal_obj = buildmeta['images'].get('metal')
img_metal = None
if img_metal_obj is not None:
img_metal = os.path.join(builddir, img_metal_obj['path'])
img_src = img_metal
else:
img_src = os.path.join(builddir, buildmeta['images']['qemu']['path'])
if img_metal_obj is None:
raise Exception("ISO generation requires `metal` image")

img_metal = os.path.join(builddir, img_metal_obj['path'])
img_metal_checksum = img_metal_obj['sha256']

# Find the directory under `/usr/lib/modules/<kver>` where the
# kernel/initrd live. It will be the 2nd entity output by
@@ -158,50 +154,34 @@ def generate_iso():
live_initramfs_stamp_path = 'etc/coreos-live-initramfs'
live_initramfs_stamp = os.path.join(livetmpd, live_initramfs_stamp_path)
os.makedirs(os.path.dirname(live_initramfs_stamp), exist_ok=True)
with open(live_initramfs_stamp, 'w') as f:
with open(live_initramfs_stamp, 'w'):
pass
run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
'--quiet', '--reproducible', '--force-local',
'-D', livetmpd, '-O', live_initramfs_cpio],
input=live_initramfs_stamp_path.encode())
run_verbose(['gzip', live_initramfs_cpio])
live_initramfs_cpio = live_initramfs_cpio + '.gz'
if is_fulliso:
tmp_initramfs = os.path.join(tmpdir, 'initramfs')

if img_metal is None:
raise Exception("fulliso requires `metal` image")
metal_dest_basename = os.path.basename(img_metal) + '.xz'
metal_dest = os.path.join(tmpisoroot, metal_dest_basename)
with open(metal_dest, 'wb') as f:
run_verbose(['xz', '-c9', '-T2', img_metal], stdout=f)
os.symlink(metal_dest_basename, os.path.join(tmpisoroot, 'image-metal.xz'))

# In the fulliso case, we copy the squashfs to the ISO root.
print(f'Compressing squashfs with {squashfs_compression}')
run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
img_src, os.path.join(tmpisoroot, 'root.squashfs'), squashfs_compression])
# Just pad with NUL bytes for the ISO image. We'll truncate the
# padding off again when copying the PXE initrd.
with open(tmp_initramfs, 'wb') as fdst:
with open(initramfs, 'rb') as fsrc:
shutil.copyfileobj(fsrc, fdst)
with open(live_initramfs_cpio, 'rb') as fsrc:
shutil.copyfileobj(fsrc, fdst)
fdst.write(bytes(initrd_ignition_padding))
os.rename(tmp_initramfs, initramfs)
elif is_live:
tmp_squashfs = os.path.join(tmpdir, 'root.squashfs')
tmp_cpio = os.path.join(tmpdir, 'root.cpio')
tmp_initramfs = os.path.join(tmpdir, 'initramfs')
cpio_input_files = [os.path.basename(tmp_squashfs)]
if args.osmet:
tmp_osmet = os.path.join(tmpdir, img_metal_obj['path'] + '.osmet')
cpio_input_files += [os.path.basename(tmp_osmet)]
print(f'Generating osmet file')
run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
img_metal, tmp_osmet, img_metal_checksum])

print(f'Compressing squashfs with {squashfs_compression}')
run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
img_src, tmp_squashfs, squashfs_compression])
img_metal, tmp_squashfs, squashfs_compression])
# create a CPIO layer which holds the root squashfs and osmet binary
run_verbose(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
'--quiet', '--reproducible', '--force-local',
'-D', os.path.dirname(tmp_squashfs), '-O', tmp_cpio],
input=os.path.basename(tmp_squashfs).encode())
input=('\n'.join(cpio_input_files)).encode())
# Compression is redundant but the kernel requires it
run_verbose(['gzip', '-1', tmp_cpio])

@@ -221,7 +201,7 @@ def generate_iso():

# Read and filter kernel arguments for substituting into ISO bootloader
result = run_verbose(['/usr/lib/coreos-assembler/gf-get-kargs',
img_src], stdout=subprocess.PIPE, text=True)
img_metal], stdout=subprocess.PIPE, text=True)
kargs_array = [karg for karg in result.stdout.split()
if karg.split('=')[0] not in live_exclude_kargs]
kargs_array.append(f"coreos.liveiso={volid}")
69 changes: 69 additions & 0 deletions src/osmet-pack
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/bin/bash
set -euo pipefail

if [ ! -f /etc/cosa-supermin ]; then
dn=$(dirname "$0")
# shellcheck source=src/cmdlib.sh
. "${dn}"/cmdlib.sh

img_src=$1; shift
osmet_dest=$1; shift
checksum=$1; shift
coreinst=${1:-${OSMET_PACK_COREOS_INSTALLER:-}}

workdir=$(pwd)
TMPDIR=tmp/tmp-osmet-pack
rm -rf "${TMPDIR}"
mkdir -p "${TMPDIR}"

if [[ $img_src == *.gz || $img_src == *.xz ]]; then
img="$(basename "$img_src")"
fatal "Cannot pack osmet from $img; not an uncompressed image"
fi

set -- "${TMPDIR}/osmet.bin" "${checksum}"
if [ -n "${coreinst:-}" ]; then
cp "${coreinst}" "${TMPDIR}/coreos-installer"
set -- "$@" "${TMPDIR}/coreos-installer"
fi

# stamp it with "osmet" serial so we find it easily in the VM
runvm -drive "if=none,id=osmet,format=raw,readonly=on,file=${img_src}" \
-device virtio-blk,serial=osmet,drive=osmet -- \
/usr/lib/coreos-assembler/osmet-pack "$@"

mv "${TMPDIR}/osmet.bin" "${osmet_dest}"
rm -rf "${TMPDIR}"

exit 0
fi

# This runs inside supermin

osmet_dest=$1; shift
checksum=$1; shift
coreinst=${1:-}

set -x

# we want /dev/disk symlinks for coreos-installer
/usr/lib/systemd/systemd-udevd --daemon
/usr/sbin/udevadm trigger --settle

rootfs=/dev/disk/by-id/virtio-osmet-part4

mkdir -p /sysroot
mount -o ro "${rootfs}" /sysroot
osname=$(ls /sysroot/ostree/deploy)
deploydir=$(find "/sysroot/ostree/deploy/$osname/deploy" -mindepth 1 -maxdepth 1 -type d)
# shellcheck disable=SC1090
description=$(. "${deploydir}/etc/os-release" && echo "${PRETTY_NAME}")

if [ -z "${coreinst}" ]; then
coreinst=${deploydir}/usr/bin/coreos-installer
fi

RUST_BACKTRACE=full "${coreinst}" osmet pack /dev/disk/by-id/virtio-osmet \
--description "${description}" \
--checksum "${checksum}" \
--output "${osmet_dest}"

0 comments on commit 37a449e

Please sign in to comment.