From 6b5de9a4254d4ddef5f70e3a405327d7cb81e63d Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Thu, 18 Jul 2024 16:27:29 +0530 Subject: [PATCH 1/7] kola: Add secureboot CI test Signed-off-by: Sayan Chowdhury --- cmd/kola/options.go | 10 +++--- platform/machine/qemu/cluster.go | 10 ++++-- platform/machine/qemu/flight.go | 7 ++-- platform/machine/unprivqemu/cluster.go | 10 ++++-- platform/platform.go | 3 ++ platform/qemu.go | 44 +++++++++++++++++++++++--- 6 files changed, 68 insertions(+), 16 deletions(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 3894a6961..54735b089 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -54,7 +54,7 @@ var ( "rhcos": "v3", } - kolaDefaultBIOS = map[string]string{ + kolaDefaultFirmware = map[string]string{ "amd64-usr": "bios-256k.bin", "arm64-usr": sdk.BuildRoot() + "/images/arm64-usr/latest/flatcar_production_qemu_uefi_efi_code.fd", } @@ -87,6 +87,7 @@ func init() { sv(&kola.UpdatePayloadFile, "update-payload", "", "Path to an update payload that should be made available to tests") bv(&kola.ForceFlatcarKey, "force-flatcar-key", false, "Use the Flatcar production key to verify update payload") sv(&kola.Options.IgnitionVersion, "ignition-version", "", "Ignition version override: v2, v3") + bv(&kola.Options.EnableSecureboot, "enable-secureboot", false, "Instantiate a Secureboot Machine") iv(&kola.Options.SSHRetries, "ssh-retries", kolaSSHRetries, "Number of retries with the SSH timeout when starting the machine") dv(&kola.Options.SSHTimeout, "ssh-timeout", kolaSSHTimeout, "A timeout for a single try of establishing an SSH connection when starting the machine") @@ -222,8 +223,9 @@ func init() { // QEMU-specific options sv(&kola.QEMUOptions.Board, "board", defaultTargetBoard, "target board") sv(&kola.QEMUOptions.DiskImage, "qemu-image", "", "path to CoreOS disk image") - sv(&kola.QEMUOptions.BIOSImage, "qemu-bios", "", "BIOS to use for QEMU vm") + sv(&kola.QEMUOptions.Firmware, "qemu-firmware", "", "firmware image to use for QEMU vm") sv(&kola.QEMUOptions.VNC, "qemu-vnc", "", "VNC port (0 for 5900, 1 for 5901, etc.)") + sv(&kola.QEMUOptions.OVMFVars, "qemu-ovmf-vars", "", "OVMF vars file to use for QEMU vm") bv(&kola.QEMUOptions.UseVanillaImage, "qemu-skip-mangle", false, "don't modify CL disk image to capture console log") sv(&kola.QEMUOptions.ExtraBaseDiskSize, "qemu-grow-base-disk-by", "", "grow base disk by the given size in bytes, following optional 1024-based suffixes are allowed: b (ignored), k, K, M, G, T") bv(&kola.QEMUOptions.EnableTPM, "qemu-tpm", false, "enable TPM device in QEMU. Requires installing swtpm. Use only with 'kola spawn', test cases are responsible for creating a VM with TPM explicitly.") @@ -310,8 +312,8 @@ func syncOptions() error { kola.QEMUOptions.DiskImage = image } - if kola.QEMUOptions.BIOSImage == "" { - kola.QEMUOptions.BIOSImage = kolaDefaultBIOS[kola.QEMUOptions.Board] + if kola.QEMUOptions.Firmware == "" { + kola.QEMUOptions.Firmware = kolaDefaultFirmware[kola.QEMUOptions.Board] } units, _ := root.PersistentFlags().GetStringSlice("debug-systemd-units") for _, unit := range units { diff --git a/platform/machine/qemu/cluster.go b/platform/machine/qemu/cluster.go index 0072b171d..6c2ffcbf4 100644 --- a/platform/machine/qemu/cluster.go +++ b/platform/machine/qemu/cluster.go @@ -133,11 +133,15 @@ ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos // This uses path arguments with path values being // relative to the folder created for this machine - biosImage, err := filepath.Abs(qc.flight.opts.BIOSImage) + firmware, err := filepath.Abs(qc.flight.opts.Firmware) if err != nil { - return nil, fmt.Errorf("failed to canonicalize bios path: %v", err) + return nil, fmt.Errorf("failed to canonicalize firmware path: %v", err) } - qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, biosImage, qm.consolePath, confPath, qc.flight.diskImagePath, conf.IsIgnition(), options) + ovmfVars, err := filepath.Abs(qc.flight.opts.OVMFVars) + if err != nil { + return nil, fmt.Errorf("failed to canonicalize ovmf vars path: %v", err) + } + qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, firmware, ovmfVars, qm.consolePath, confPath, qc.flight.diskImagePath, qc.flight.opts.EnableSecureboot, conf.IsIgnition(), options) if err != nil { return nil, err } diff --git a/platform/machine/qemu/flight.go b/platform/machine/qemu/flight.go index 4af63b623..c638fc400 100644 --- a/platform/machine/qemu/flight.go +++ b/platform/machine/qemu/flight.go @@ -35,9 +35,12 @@ type Options struct { // DiskImage is the full path to the disk image to boot in QEMU. DiskImage string - // BIOSImage is name of the BIOS file to pass to QEMU. + // Firmware is name of the Firmware file to pass to QEMU. // It can be a plain name, or a full path. - BIOSImage string + Firmware string + + // OMVF Vars file to pass to QEMU UEFI + OVMFVars string // Don't modify CL disk images to add console logging UseVanillaImage bool diff --git a/platform/machine/unprivqemu/cluster.go b/platform/machine/unprivqemu/cluster.go index 6b80f38bf..7d9c33d4f 100644 --- a/platform/machine/unprivqemu/cluster.go +++ b/platform/machine/unprivqemu/cluster.go @@ -145,11 +145,15 @@ LinkLocalAddressing=no } // This uses path arguments with path values being // relative to the folder created for this machine - biosImage, err := filepath.Abs(qc.flight.opts.BIOSImage) + firmware, err := filepath.Abs(qc.flight.opts.Firmware) if err != nil { - return nil, fmt.Errorf("failed to canonicalize bios path: %v", err) + return nil, fmt.Errorf("failed to canonicalize firmware path: %v", err) } - qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, biosImage, qm.consolePath, confPath, qc.flight.diskImagePath, conf.IsIgnition(), options) + ovmfVars, err := filepath.Abs(qc.flight.opts.OVMFVars) + if err != nil { + return nil, fmt.Errorf("failed to canonicalize ovmf vars path: %v", err) + } + qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, firmware, ovmfVars, qm.consolePath, confPath, qc.flight.diskImagePath, qc.flight.opts.EnableSecureboot, conf.IsIgnition(), options) if err != nil { return nil, err } diff --git a/platform/platform.go b/platform/platform.go index dd9c01e28..d75bbafc8 100644 --- a/platform/platform.go +++ b/platform/platform.go @@ -163,6 +163,9 @@ type Options struct { // Board is the board used by the image Board string + // Toggle to instantiate a secureboot instance. + EnableSecureboot bool + // How many times to retry establishing an SSH connection when // creating a journal or when doing a machine check. SSHRetries int diff --git a/platform/qemu.go b/platform/qemu.go index 1475bee4c..a5e01662e 100644 --- a/platform/qemu.go +++ b/platform/qemu.go @@ -17,12 +17,14 @@ package platform import ( "errors" "fmt" + "io" "io/ioutil" "os" origExec "os/exec" "path/filepath" "regexp" "runtime" + "slices" "strconv" "strings" "time" @@ -293,7 +295,7 @@ func mkpath(basedir string) (string, error) { return f.Name(), nil } -func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImagePath string, isIgnition bool, options MachineOptions) ([]string, []*os.File, error) { +func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, diskImagePath string, enableSecureboot, isIgnition bool, options MachineOptions) ([]string, []*os.File, error) { var qmCmd []string // As we expand this list of supported native + board @@ -307,7 +309,7 @@ func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImageP qmBinary = "qemu-system-x86_64" qmCmd = []string{ "qemu-system-x86_64", - "-machine", "accel=kvm", + "-machine", "q35,accel=kvm,smm=on", "-cpu", "host", "-m", "2512", } @@ -340,7 +342,6 @@ func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImageP } qmCmd = append(qmCmd, - "-bios", biosImage, "-smp", "4", "-uuid", uuid, "-display", "none", @@ -348,8 +349,38 @@ func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImageP "-serial", "chardev:log", "-object", "rng-random,filename=/dev/urandom,id=rng0", "-device", "virtio-rng-pci,rng=rng0", + "-drive", fmt.Sprintf("if=pflash,unit=0,file=%v,format=raw,readonly=on", firmware), ) + if enableSecureboot == true { + // Create a copy of the OVMF Vars + ovmfVarsSrc, err := os.Open(ovmfVars) + if err != nil { + return nil, nil, err + } + defer ovmfVarsSrc.Close() + + ovmfVarsCopy, err := ioutil.TempFile("/var/tmp/", "mantle-qemu") + if err != nil { + return nil, nil, err + } + + if _, err := io.Copy(ovmfVarsCopy, ovmfVarsSrc); err != nil { + return nil, nil, err + } + + _, err = ovmfVarsCopy.Seek(0, 0) + if err != nil { + return nil, nil, err + } + + qmCmd = append(qmCmd, + "-global", "ICH9-LPC.disable_s3=1", + "-global", "driver=cfi.pflash01,property=secure,value=on", + "-drive", fmt.Sprintf("if=pflash,unit=1,file=%v,format=raw", ovmfVarsCopy.Name()), + ) + } + if options.EnableTPM { var tpm string switch board { @@ -413,6 +444,11 @@ func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImageP fdset := 1 for _, disk := range allDisks { + bootIndexArg := "" + if slices.Contains(disk.DeviceOpts, "serial=primary-disk") { + bootIndexArg = ",bootindex=1" + } + optionsDiskFile, err := disk.setupFile() if err != nil { return nil, nil, err @@ -423,7 +459,7 @@ func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImageP id := fmt.Sprintf("d%d", fdnum) qmCmd = append(qmCmd, "-add-fd", fmt.Sprintf("fd=%d,set=%d", fdnum, fdset), "-drive", fmt.Sprintf("if=none,id=%s,format=qcow2,file=/dev/fdset/%d%s", id, fdset, autoReadOnly), - "-device", Virtio(board, "blk", fmt.Sprintf("drive=%s%s", id, disk.getOpts()))) + "-device", Virtio(board, "blk", fmt.Sprintf("drive=%s%s%s", id, disk.getOpts(), bootIndexArg))) fdnum += 1 fdset += 1 } From 606b63a1cf7a781e98a2ac94a91adac8d2462b5b Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 4 Sep 2024 17:05:42 +0200 Subject: [PATCH 2/7] kola: Tweak ovmfvar/sboot handling Continue supporting BIOS by passing `-bios` and only enable `smm=on` when secure boot is requested, as it requires build of OVMF code. This special build is required for secure boot support, but non-sboot OVMFs won't support it. --- cmd/kola/options.go | 3 +++ platform/qemu.go | 27 ++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 54735b089..84edc1197 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -315,6 +315,9 @@ func syncOptions() error { if kola.QEMUOptions.Firmware == "" { kola.QEMUOptions.Firmware = kolaDefaultFirmware[kola.QEMUOptions.Board] } + if kola.QEMUOptions.EnableSecureboot && kola.QEMUOptions.OVMFVars == "" { + return fmt.Errorf("Secureboot requires OVMF vars file") + } units, _ := root.PersistentFlags().GetStringSlice("debug-systemd-units") for _, unit := range units { kola.Options.SystemdDropins = append(kola.Options.SystemdDropins, platform.SystemdDropin{ diff --git a/platform/qemu.go b/platform/qemu.go index a5e01662e..3e556e68f 100644 --- a/platform/qemu.go +++ b/platform/qemu.go @@ -304,12 +304,16 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d // similar in cosa run var qmBinary string combo := runtime.GOARCH + "--" + board + smmFlag := "" + if enableSecureboot { + smmFlag = ",smm=on" + } switch combo { case "amd64--amd64-usr": qmBinary = "qemu-system-x86_64" qmCmd = []string{ "qemu-system-x86_64", - "-machine", "q35,accel=kvm,smm=on", + "-machine", fmt.Sprintf("q35,accel=kvm%s", smmFlag), "-cpu", "host", "-m", "2512", } @@ -341,6 +345,11 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d panic("host-guest combo not supported: " + combo) } + if ovmfVars == "" { + qmCmd = append(qmCmd, + "-bios", firmware, + ) + } qmCmd = append(qmCmd, "-smp", "4", "-uuid", uuid, @@ -349,10 +358,14 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d "-serial", "chardev:log", "-object", "rng-random,filename=/dev/urandom,id=rng0", "-device", "virtio-rng-pci,rng=rng0", - "-drive", fmt.Sprintf("if=pflash,unit=0,file=%v,format=raw,readonly=on", firmware), ) + if ovmfVars != "" { + qmCmd = append(qmCmd, + "-drive", fmt.Sprintf("if=pflash,unit=0,file=%v,format=raw,readonly=on", firmware), + ) + } - if enableSecureboot == true { + if ovmfVars != "" { // Create a copy of the OVMF Vars ovmfVarsSrc, err := os.Open(ovmfVars) if err != nil { @@ -374,9 +387,13 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d return nil, nil, err } + if enableSecureboot { + qmCmd = append(qmCmd, + "-global", "ICH9-LPC.disable_s3=1", + "-global", "driver=cfi.pflash01,property=secure,value=on", + ) + } qmCmd = append(qmCmd, - "-global", "ICH9-LPC.disable_s3=1", - "-global", "driver=cfi.pflash01,property=secure,value=on", "-drive", fmt.Sprintf("if=pflash,unit=1,file=%v,format=raw", ovmfVarsCopy.Name()), ) } From 0a723ec12141f43a08f757eb8fa2c82fab8b2961 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 6 Sep 2024 17:26:23 +0200 Subject: [PATCH 3/7] qemu: Create OVMF vars copy in instance dir and cleanup on shutdown. Signed-off-by: Jeremi Piotrowski --- platform/machine/qemu/cluster.go | 18 ++++++++--- platform/machine/qemu/machine.go | 9 ++++++ platform/machine/unprivqemu/cluster.go | 19 +++++++++--- platform/machine/unprivqemu/machine.go | 9 ++++++ platform/qemu.go | 42 +++++++++----------------- 5 files changed, 61 insertions(+), 36 deletions(-) diff --git a/platform/machine/qemu/cluster.go b/platform/machine/qemu/cluster.go index 6c2ffcbf4..26c1bd366 100644 --- a/platform/machine/qemu/cluster.go +++ b/platform/machine/qemu/cluster.go @@ -17,6 +17,7 @@ package qemu import ( "fmt" "os" + "path" "path/filepath" "strings" "sync" @@ -137,10 +138,19 @@ ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos if err != nil { return nil, fmt.Errorf("failed to canonicalize firmware path: %v", err) } - ovmfVars, err := filepath.Abs(qc.flight.opts.OVMFVars) - if err != nil { - return nil, fmt.Errorf("failed to canonicalize ovmf vars path: %v", err) + ovmfVars := "" + if qc.flight.opts.OVMFVars != "" { + ovmfVars, err = platform.CreateOvmfVarsCopy(qm.subDir, qc.flight.opts.OVMFVars) + if err != nil { + return nil, err + } + defer func() { + if ovmfVars != "" { + os.Remove(path.Join(qm.subDir, ovmfVars)) + } + }() } + qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, firmware, ovmfVars, qm.consolePath, confPath, qc.flight.diskImagePath, qc.flight.opts.EnableSecureboot, conf.IsIgnition(), options) if err != nil { return nil, err @@ -185,7 +195,7 @@ ExecStartPost=/usr/bin/ln -fs /run/metadata/flatcar /run/metadata/coreos // from this point on Destroy() is responsible for cleaning up swtpm qm.swtpm, swtpm = swtpm, nil - + qm.ovmfVars, ovmfVars = ovmfVars, "" plog.Debugf("qemu PID (manual cleanup needed if --remove=false): %v", qm.qemu.Pid()) if err := platform.StartMachine(qm, qm.journal); err != nil { diff --git a/platform/machine/qemu/machine.go b/platform/machine/qemu/machine.go index 35083ac9f..4d455738c 100644 --- a/platform/machine/qemu/machine.go +++ b/platform/machine/qemu/machine.go @@ -16,6 +16,8 @@ package qemu import ( "io/ioutil" + "os" + "path" "path/filepath" "golang.org/x/crypto/ssh" @@ -33,6 +35,7 @@ type machine struct { journal *platform.Journal consolePath string console string + ovmfVars string subDir string swtpm *local.SoftwareTPM } @@ -76,6 +79,12 @@ func (m *machine) Destroy() { if m.swtpm != nil { m.swtpm.Stop() } + if m.ovmfVars != "" { + err := os.Remove(path.Join(m.subDir, m.ovmfVars)) + if err != nil { + plog.Errorf("Error removing OVMF vars: %v", err) + } + } m.journal.Destroy() if buf, err := ioutil.ReadFile(filepath.Join(m.subDir, m.consolePath)); err == nil { diff --git a/platform/machine/unprivqemu/cluster.go b/platform/machine/unprivqemu/cluster.go index 7d9c33d4f..dc5ebfcbe 100644 --- a/platform/machine/unprivqemu/cluster.go +++ b/platform/machine/unprivqemu/cluster.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "net" "os" + "path" "path/filepath" "regexp" "strconv" @@ -143,16 +144,25 @@ LinkLocalAddressing=no } }() } + + ovmfVars := "" + if qc.flight.opts.OVMFVars != "" { + ovmfVars, err = platform.CreateOvmfVarsCopy(qm.subDir, qc.flight.opts.OVMFVars) + if err != nil { + return nil, err + } + defer func() { + if ovmfVars != "" { + os.Remove(path.Join(qm.subDir, ovmfVars)) + } + }() + } // This uses path arguments with path values being // relative to the folder created for this machine firmware, err := filepath.Abs(qc.flight.opts.Firmware) if err != nil { return nil, fmt.Errorf("failed to canonicalize firmware path: %v", err) } - ovmfVars, err := filepath.Abs(qc.flight.opts.OVMFVars) - if err != nil { - return nil, fmt.Errorf("failed to canonicalize ovmf vars path: %v", err) - } qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, firmware, ovmfVars, qm.consolePath, confPath, qc.flight.diskImagePath, qc.flight.opts.EnableSecureboot, conf.IsIgnition(), options) if err != nil { return nil, err @@ -186,6 +196,7 @@ LinkLocalAddressing=no // from this point on Destroy() is responsible for cleaning up swtpm qm.swtpm, swtpm = swtpm, nil + qm.ovmfVars, ovmfVars = ovmfVars, "" plog.Debugf("qemu PID (manual cleanup needed if --remove=false): %v", qm.qemu.Pid()) pid := strconv.Itoa(qm.qemu.Pid()) diff --git a/platform/machine/unprivqemu/machine.go b/platform/machine/unprivqemu/machine.go index 219d8fc17..da060e848 100644 --- a/platform/machine/unprivqemu/machine.go +++ b/platform/machine/unprivqemu/machine.go @@ -16,6 +16,8 @@ package unprivqemu import ( "io/ioutil" + "os" + "path" "path/filepath" "golang.org/x/crypto/ssh" @@ -32,6 +34,7 @@ type machine struct { journal *platform.Journal consolePath string console string + ovmfVars string subDir string swtpm *local.SoftwareTPM ip string @@ -78,6 +81,12 @@ func (m *machine) Destroy() { if m.swtpm != nil { m.swtpm.Stop() } + if m.ovmfVars != "" { + err := os.Remove(path.Join(m.subDir, m.ovmfVars)) + if err != nil { + plog.Errorf("Error removing OVMF vars: %v", err) + } + } m.journal.Destroy() if buf, err := ioutil.ReadFile(filepath.Join(m.subDir, m.consolePath)); err == nil { diff --git a/platform/qemu.go b/platform/qemu.go index 3e556e68f..3a6efb698 100644 --- a/platform/qemu.go +++ b/platform/qemu.go @@ -17,10 +17,10 @@ package platform import ( "errors" "fmt" - "io" "io/ioutil" "os" origExec "os/exec" + "path" "path/filepath" "regexp" "runtime" @@ -31,6 +31,7 @@ import ( "github.com/coreos/go-semver/semver" + "github.com/flatcar/mantle/system" "github.com/flatcar/mantle/system/exec" "github.com/flatcar/mantle/util" ) @@ -295,6 +296,17 @@ func mkpath(basedir string) (string, error) { return f.Name(), nil } +// returns basename of the copied file because it is intended +// to be used by qemu, which will be started in dir +func CreateOvmfVarsCopy(dir, ovmfVars string) (string, error) { + ovmfVarsDst := path.Join(dir, path.Base(ovmfVars)) + err := system.CopyRegularFile(ovmfVars, ovmfVarsDst) + if err != nil { + return "", err + } + return path.Base(ovmfVars), nil +} + func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, diskImagePath string, enableSecureboot, isIgnition bool, options MachineOptions) ([]string, []*os.File, error) { var qmCmd []string @@ -362,40 +374,14 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d if ovmfVars != "" { qmCmd = append(qmCmd, "-drive", fmt.Sprintf("if=pflash,unit=0,file=%v,format=raw,readonly=on", firmware), + "-drive", fmt.Sprintf("if=pflash,unit=1,file=%v,format=raw", ovmfVars), ) - } - - if ovmfVars != "" { - // Create a copy of the OVMF Vars - ovmfVarsSrc, err := os.Open(ovmfVars) - if err != nil { - return nil, nil, err - } - defer ovmfVarsSrc.Close() - - ovmfVarsCopy, err := ioutil.TempFile("/var/tmp/", "mantle-qemu") - if err != nil { - return nil, nil, err - } - - if _, err := io.Copy(ovmfVarsCopy, ovmfVarsSrc); err != nil { - return nil, nil, err - } - - _, err = ovmfVarsCopy.Seek(0, 0) - if err != nil { - return nil, nil, err - } - if enableSecureboot { qmCmd = append(qmCmd, "-global", "ICH9-LPC.disable_s3=1", "-global", "driver=cfi.pflash01,property=secure,value=on", ) } - qmCmd = append(qmCmd, - "-drive", fmt.Sprintf("if=pflash,unit=1,file=%v,format=raw", ovmfVarsCopy.Name()), - ) } if options.EnableTPM { From 101bc3a11d4a7a980a06b25851157b3f671a1d2a Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Mon, 9 Sep 2024 16:43:17 +0200 Subject: [PATCH 4/7] kola: Add qemu-bios fallback To make this change easier to apply to all channels. Signed-off-by: Jeremi Piotrowski --- cmd/kola/options.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 84edc1197..3b3c12070 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -223,6 +223,8 @@ func init() { // QEMU-specific options sv(&kola.QEMUOptions.Board, "board", defaultTargetBoard, "target board") sv(&kola.QEMUOptions.DiskImage, "qemu-image", "", "path to CoreOS disk image") + sv(&kola.QEMUOptions.Firmware, "qemu-bios", "", "bios to use for QEMU vm") + root.PersistentFlags().MarkDeprecated("qemu-bios", "use --qemu-firmware") sv(&kola.QEMUOptions.Firmware, "qemu-firmware", "", "firmware image to use for QEMU vm") sv(&kola.QEMUOptions.VNC, "qemu-vnc", "", "VNC port (0 for 5900, 1 for 5901, etc.)") sv(&kola.QEMUOptions.OVMFVars, "qemu-ovmf-vars", "", "OVMF vars file to use for QEMU vm") From 408e861fd446b7e9743809d6f39ffe14e908b611 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Tue, 10 Sep 2024 06:42:32 +0200 Subject: [PATCH 5/7] tests: Skip kmod tests when secure boot is enabled Kernel lockdown blocks loading unsigned kernel modules, so these tests need to be disabled. Eventually the zfs sysext should ship a signed kernel modules, but falco is built on the running system and won't work the same way. Falco suggests running in eBPF mode instead. --- kola/harness.go | 4 ++++ kola/tests/misc/falco.go | 4 +++- kola/tests/sysext/zfs.go | 11 ++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/kola/harness.go b/kola/harness.go index eb28787b7..d8df8a5de 100644 --- a/kola/harness.go +++ b/kola/harness.go @@ -223,6 +223,10 @@ var ( } ) +func SkipSecureboot(_ semver.Version, channel, arch, platform string) bool { + return Options.EnableSecureboot +} + // NativeRunner is a closure passed to all kola test functions and used // to run native go functions directly on kola machines. It is necessary // glue until kola does introspection. diff --git a/kola/tests/misc/falco.go b/kola/tests/misc/falco.go index 2fddbb06b..34189e73a 100644 --- a/kola/tests/misc/falco.go +++ b/kola/tests/misc/falco.go @@ -1,6 +1,7 @@ package misc import ( + "github.com/flatcar/mantle/kola" "github.com/flatcar/mantle/kola/cluster" "github.com/flatcar/mantle/kola/register" ) @@ -16,7 +17,8 @@ func init() { // falco builder container can't handle our arm64 config (yet) Architectures: []string{"amd64"}, // selinux blocks insmod from within container - Flags: []register.Flag{register.NoEnableSelinux}, + Flags: []register.Flag{register.NoEnableSelinux}, + SkipFunc: kola.SkipSecureboot, }) } diff --git a/kola/tests/sysext/zfs.go b/kola/tests/sysext/zfs.go index 09a444c7b..7b6c0d1ca 100644 --- a/kola/tests/sysext/zfs.go +++ b/kola/tests/sysext/zfs.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/coreos/go-semver/semver" + "github.com/flatcar/mantle/kola" "github.com/flatcar/mantle/kola/cluster" "github.com/flatcar/mantle/kola/register" "github.com/flatcar/mantle/platform" @@ -133,7 +134,7 @@ func init() { // This test is normally not related to the cloud environment Platforms: []string{"qemu", "qemu-unpriv"}, MinVersion: semver.Version{Major: 3902}, - SkipFunc: skipOnGha, + SkipFunc: skipZfs, }) register.Register(®ister.Test{ @@ -144,7 +145,7 @@ func init() { // This test is normally not related to the cloud environment Platforms: []string{"qemu", "qemu-unpriv"}, MinVersion: semver.Version{Major: 3902}, - SkipFunc: skipOnGha, + SkipFunc: skipZfs, }) register.Register(®ister.Test{ @@ -155,10 +156,14 @@ func init() { // This test is normally not related to the cloud environment Platforms: []string{"qemu", "qemu-unpriv"}, MinVersion: semver.Version{Major: 3902}, - SkipFunc: skipOnGha, + SkipFunc: skipZfs, }) } +func skipZfs(version semver.Version, channel, arch, platform string) bool { + return kola.SkipSecureboot(version, channel, arch, platform) || skipOnGha(version, channel, arch, platform) +} + func skipOnGha(version semver.Version, channel, arch, platform string) bool { // Skip for release tests as we don't yet have a sysext signed with the // prod key, nor is it on the release server. From 6dc4d85bf6ad3688c41cf582cc6ac552adc4f30a Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Tue, 10 Sep 2024 17:44:46 +0200 Subject: [PATCH 6/7] platform/qemu: Add comment about s3 disabling Signed-off-by: Jeremi Piotrowski --- platform/qemu.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platform/qemu.go b/platform/qemu.go index 3a6efb698..8c80c1f06 100644 --- a/platform/qemu.go +++ b/platform/qemu.go @@ -377,6 +377,10 @@ func CreateQEMUCommand(board, uuid, firmware, ovmfVars, consolePath, confPath, d "-drive", fmt.Sprintf("if=pflash,unit=1,file=%v,format=raw", ovmfVars), ) if enableSecureboot { + // When OVMF is built for X64 with SMM enabled S3 (suspend/resume) + // must be disabled. This is required for secure boot and not very + // well documented. The flag comes from here: + // https://github.com/tianocore/edk2/blob/b81557a00c61cc80ab118828f16ed9ce79455880/OvmfPkg/README#L213 qmCmd = append(qmCmd, "-global", "ICH9-LPC.disable_s3=1", "-global", "driver=cfi.pflash01,property=secure,value=on", From b496a0b6a747d2247bd29fe6034f0014293ddf12 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 11 Sep 2024 10:47:27 +0200 Subject: [PATCH 7/7] README: Update example of running kola The previous instructions still work, but update to show the newly added cli options. Signed-off-by: Jeremi Piotrowski --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d860dd0fa..0b979262f 100644 --- a/README.md +++ b/README.md @@ -95,8 +95,13 @@ wget https://alpha.release.flatcar-linux.net/amd64-usr/current/flatcar_productio wget https://alpha.release.flatcar-linux.net/amd64-usr/current/flatcar_production_qemu_uefi_efi_code.fd.sig gpg --verify flatcar_production_qemu_uefi_efi_code.fd.sig +wget https://alpha.release.flatcar-linux.net/amd64-usr/current/flatcar_production_qemu_uefi_efi_vars.fd +wget https://alpha.release.flatcar-linux.net/amd64-usr/current/flatcar_production_qemu_uefi_efi_vars.fd.sig +gpg --verify flatcar_production_qemu_uefi_efi_vars.fd.sig + sudo ./bin/kola run --board amd64-usr --key ${HOME}/.ssh/id_rsa.pub -k -b cl -p qemu \ - --qemu-bios flatcar_production_qemu_uefi_efi_code.fd \ + --qemu-firmware flatcar_production_qemu_uefi_efi_code.fd \ + --qemu-ovmf-vars flatcar_production_qemu_uefi_efi_vars.fd \ --qemu-image flatcar_production_qemu_image.img \ cl.locksmith.cluster ``` @@ -104,17 +109,17 @@ sudo ./bin/kola run --board amd64-usr --key ${HOME}/.ssh/id_rsa.pub -k -b cl -p ###### Run tests for ARM64 Example with the latest `alpha` release: ```shell -wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_secure_image.img -wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_secure_image.img.sig -gpg --verify flatcar_production_qemu_uefi_secure_image.img.sig +wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_image.img +wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_image.img.sig +gpg --verify flatcar_production_qemu_uefi_image.img.sig wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_efi_code.fd wget https://alpha.release.flatcar-linux.net/arm64-usr/current/flatcar_production_qemu_uefi_efi_code.fd.sig gpg --verify flatcar_production_qemu_uefi_efi_code.fd.sig sudo ./bin/kola run --board arm64-usr --key ${HOME}/.ssh/id_rsa.pub -k -b cl -p qemu \ - --qemu-bios flatcar_production_qemu_uefi_efi_code.fd \ - --qemu-image flatcar_production_qemu_uefi_secure_image.bin \ + --qemu-firmware flatcar_production_qemu_uefi_efi_code.fd \ + --qemu-image flatcar_production_qemu_uefi_image.img \ cl.etcd-member.discovery ```