Add NixOS image for bare-metal Kata

This switches the image used in our bare-metal Kata uses (e.g. non-AKS and non-peerpods) to a NixOS image that we build in-tree as a MicroVM image (e.g. separated kernel, initrd, cmdline and rootfs).

docs/docs/features-limitations.md

@@ -34,3 +34,10 @@ Currently, this requires inspecting the iptables rules on startup or terminating
 The Contrast Coordinator is a singleton and can't be scaled to more than one instance.
 When this instance's pod is restarted, for example for node maintenance, it needs to be recovered manually.
 In a future release, we plan to support distributed Coordinator instances that can recover automatically.
+
+## Overriding Kata configuration
+
+Kata Containers supports [overriding certain configuration values via Kubernetes annotations](https://github.com/kata-containers/kata-containers/blob/b4da4b5e3b9b21048af9333b071235a57a3e9493/docs/how-to/how-to-set-sandbox-config-kata.md).
+
+It needs to be noted that setting these values is unsupported, and doing so may lead to unexpected
+behaviour, as Contrast isn't tested against all possible configuration combinations.

nodeinstaller/internal/constants/configuration-qemu-tdx.toml

@@ -18,6 +18,9 @@ cpu_features="-vmx-rdseed-exit,pmu=off"
 default_vcpus = 1
 default_maxvcpus = 0
 default_bridges = 1
+# On TDX, when lowering this, the patch:
+# packages/by-name/qemu-tdx-static/0004-hw-x86-load-initrd-to-static-address.patch
+# needs to be updated accordingly.
 default_memory = 2048
 default_maxmemory = 0
 disable_block_device_use = false

nodeinstaller/internal/constants/constants.go

@@ -64,37 +64,36 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, qemuExtraKer
 		config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "tdx", "share", "OVMF.fd")
 		config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
-		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")}
 		config.Hypervisor["qemu"]["block_device_aio"] = "threads"
 		config.Hypervisor["qemu"]["shared_fs"] = "none"
-		kernelParams := qemuExtraKernelParams
+		config.Hypervisor["qemu"]["initrd"] = filepath.Join(baseDir, "share", "kata-initrd.zst")
+		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
+		// Replace the kernel params entirely (and don't append) since that's
+		// also what we do when calculating the launch measurement.
+		config.Hypervisor["qemu"]["kernel_params"] = qemuExtraKernelParams
 		if debug {
 			config.Hypervisor["qemu"]["enable_debug"] = true
 		}
-		// Replace the kernel params entirely (and don't append) since that's
-		// also what we do when calculating the launch measurement.
-		config.Hypervisor["qemu"]["kernel_params"] = kernelParams
 	case platforms.K3sQEMUSNP:
 		if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
 		}
 		config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "snp", "share", "OVMF.fd")
 		config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
-		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
-		delete(config.Hypervisor["qemu"], "initrd")
 		config.Hypervisor["qemu"]["block_device_aio"] = "threads"
 		config.Hypervisor["qemu"]["shared_fs"] = "none"
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")}
 		config.Hypervisor["qemu"]["rootfs_type"] = "erofs"
-		kernelParams := qemuExtraKernelParams
+		config.Hypervisor["qemu"]["initrd"] = filepath.Join(baseDir, "share", "kata-initrd.zst")
+		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
+		// Replace the kernel params entirely (and don't append) since that's
+		// also what we do when calculating the launch measurement.
+		config.Hypervisor["qemu"]["kernel_params"] = qemuExtraKernelParams
 		if debug {
 			config.Hypervisor["qemu"]["enable_debug"] = true
 		}
-		// Replace the kernel params entirely (and don't append) since that's
-		// also what we do when calculating the launch measurement.
-		config.Hypervisor["qemu"]["kernel_params"] = kernelParams
 	default:
 		return nil, fmt.Errorf("unsupported platform: %s", platform)
 	}

packages/by-name/OVMF-TDX/package.nix

@@ -9,7 +9,7 @@
   debug ? false,
 }:
 
-edk2.mkDerivation "OvmfPkg/IntelTdx/IntelTdxX64.dsc" rec {
+edk2.mkDerivation "OvmfPkg/IntelTdx/IntelTdxX64.dsc" {
   name = "OVMF-TDX";
 
   buildFlags = lib.optionals debug [ "-D DEBUG_ON_SERIAL_PORT=TRUE" ];

packages/by-name/image-podvm/package.nix

@@ -15,5 +15,6 @@ buildVerityUKI (mkNixosConfig {
     debug.enable = withDebug;
     gpu.enable = withGPU;
     azure.enable = withCSP == "azure";
+    peerpods.enable = true;
   };
 })

packages/by-name/kata/contrast-node-installer-image/package.nix

@@ -47,6 +47,10 @@ let
               url = "file:///opt/edgeless/share/kata-kernel";
               path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel";
             }
+            {
+              url = "file:///opt/edgeless/share/kata-initrd.zst";
+              path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst";
+            }
             {
               url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64";
               path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64";
@@ -106,7 +110,7 @@ let
             }
           ];
           inherit debugRuntime;
-          qemuExtraKernelParams = kata.snp-launch-digest.dmVerityArgs;
+          qemuExtraKernelParams = kata.kata-image.cmdline;
         };
         destination = "/config/contrast-node-install.json";
       }
@@ -116,13 +120,17 @@ let
   kata-container-img = ociLayerTar {
     files = [
       {
-        source = kata.kata-image;
+        source = "${kata.kata-image}/${kata.kata-image.imageFileName}";
         destination = "/opt/edgeless/share/kata-containers.img";
       }
       {
-        source = "${kata.kata-kernel-uvm}/bzImage";
+        source = "${kata.kata-image.kernel}/bzImage";
         destination = "/opt/edgeless/share/kata-kernel";
       }
+      {
+        source = "${kata.kata-image.initialRamdisk}/initrd";
+        destination = "/opt/edgeless/share/kata-initrd.zst";
+      }
     ];
   };