From b34ccb30138cc2d05145fff6902d9ee399447586 Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Thu, 15 Oct 2020 17:17:09 +0200 Subject: [PATCH] allow to customize vm devices Signed-off-by: Marcin Franczyk --- pkg/kubevirt/apis/provider_spec.go | 23 +++++++++ pkg/kubevirt/core/core.go | 15 ++++-- pkg/kubevirt/core/core_test.go | 74 ++++++++++++++++++++++++++- pkg/kubevirt/core/util.go | 56 +++++++++++++------- pkg/kubevirt/validation/validation.go | 42 +++++++++++++++ 5 files changed, 185 insertions(+), 25 deletions(-) diff --git a/pkg/kubevirt/apis/provider_spec.go b/pkg/kubevirt/apis/provider_spec.go index 8340619..7e825d0 100644 --- a/pkg/kubevirt/apis/provider_spec.go +++ b/pkg/kubevirt/apis/provider_spec.go @@ -20,6 +20,9 @@ import ( cdicorev1alpha1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" ) +// RootDiskName is name of the root disk +const RootDiskName = "root-disk" + // KubeVirtProviderSpec is the kubevirt provider specification. // It contains parameters to be used when creating kubevirt VMs. type KubeVirtProviderSpec struct { @@ -29,6 +32,8 @@ type KubeVirtProviderSpec struct { Zone string `json:"zone"` // Resources specifies the requests and limits for VM resources (CPU and memory). Resources kubevirtv1.ResourceRequirements `json:"resources"` + // Devices is the specification of disks and additional high performance options + Devices *Devices `json:"devices,omitempty"` // RootVolume is the specification for the root volume of the VM. RootVolume cdicorev1alpha1.DataVolumeSpec `json:"rootVolume"` // AdditionalVolumes is an optional list of additional volumes attached to the VM. @@ -64,6 +69,8 @@ type KubeVirtProviderSpec struct { // AdditionalVolumeSpec represents an additional volume attached to a VM. // Only one of its members may be specified. type AdditionalVolumeSpec struct { + // Name is the additional volume name + Name string `json:"name"` // DataVolume is an optional specification of an additional data volume. // +optional DataVolume *cdicorev1alpha1.DataVolumeSpec `json:"dataVolume,omitempty"` @@ -89,6 +96,22 @@ type VolumeSource struct { Secret *kubevirtv1.SecretVolumeSource `json:"secret,omitempty"` } +// Devices allows to fine-tune devices attached to KubeVirt VM +type Devices struct { + // Disks allows customizing the disks attached to the VM. + // +optional + Disks []kubevirtv1.Disk `json:"disks,omitempty"` + // Rng specifies whether to have a random number generator from host. + // +optional + Rng *kubevirtv1.Rng `json:"rng,omitempty"` + // BlockMultiQueue specifies whether to enable virtio multi-queue for block devices. + // +optional + BlockMultiQueue bool `json:"blockMultiQueue,omitempty"` + // NetworkInterfaceMultiQueue specifies whether virtual network interfaces configured with a virtio bus will also enable the vhost multi-queue feature. + // +optional + NetworkInterfaceMultiQueue bool `json:"networkInterfaceMultiqueue,omitempty"` +} + // NetworkSpec contains information about a network. type NetworkSpec struct { // Name is the name (in the format or /) of the network. diff --git a/pkg/kubevirt/core/core.go b/pkg/kubevirt/core/core.go index 57d322c..c7b8b6f 100644 --- a/pkg/kubevirt/core/core.go +++ b/pkg/kubevirt/core/core.go @@ -114,9 +114,12 @@ func (p PluginSPIImpl) CreateMachine(ctx context.Context, machineName string, pr // Build interfaces and networks interfaces, networks, networkData := buildNetworks(providerSpec.Networks) + var devices api.Devices + if providerSpec.Devices != nil { + devices = *providerSpec.Devices + } // Build disks, volumes, and data volumes - disks, volumes, dataVolumes := buildVolumes(machineName, namespace, userDataSecretName, networkData, providerSpec.RootVolume, providerSpec.AdditionalVolumes) - + disks, volumes, dataVolumes := buildVolumes(machineName, namespace, userDataSecretName, networkData, providerSpec.RootVolume, providerSpec.AdditionalVolumes, devices.Disks) // Get Kubernetes version k8sVersion, err := p.svf.GetServerVersion(secret) if err != nil { @@ -160,8 +163,11 @@ func (p PluginSPIImpl) CreateMachine(ctx context.Context, machineName string, pr CPU: providerSpec.CPU, Memory: providerSpec.Memory, Devices: kubevirtv1.Devices{ - Disks: disks, - Interfaces: interfaces, + Disks: disks, + Interfaces: interfaces, + Rng: devices.Rng, + BlockMultiQueue: &devices.BlockMultiQueue, + NetworkInterfaceMultiQueue: &devices.NetworkInterfaceMultiQueue, }, }, Affinity: affinity, @@ -175,7 +181,6 @@ func (p PluginSPIImpl) CreateMachine(ctx context.Context, machineName string, pr DataVolumeTemplates: dataVolumes, }, } - // Create the VM if err := c.Create(ctx, virtualMachine); err != nil { return "", errors.Wrapf(err, "could not create VirtualMachine %q", machineName) diff --git a/pkg/kubevirt/core/core_test.go b/pkg/kubevirt/core/core_test.go index 23a73b5..9a27ba4 100644 --- a/pkg/kubevirt/core/core_test.go +++ b/pkg/kubevirt/core/core_test.go @@ -87,6 +87,29 @@ var _ = Describe("PluginSPIImpl", func() { corev1.ResourceMemory: resource.MustParse("8Gi"), }, }, + Devices: &api.Devices{ + Disks: []kubevirtv1.Disk{ + { + Name: api.RootDiskName, + DiskDevice: kubevirtv1.DiskDevice{ + Disk: &kubevirtv1.DiskTarget{ + Bus: "virtio", + }, + }, + DedicatedIOThread: pointer.BoolPtr(true), + }, + { + Name: "volume-1", + DiskDevice: kubevirtv1.DiskDevice{ + Disk: &kubevirtv1.DiskTarget{ + Bus: "virtio", + }, + }, + DedicatedIOThread: pointer.BoolPtr(true), + }, + }, + BlockMultiQueue: true, + }, RootVolume: cdicorev1alpha1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -107,6 +130,7 @@ var _ = Describe("PluginSPIImpl", func() { }, AdditionalVolumes: []api.AdditionalVolumeSpec{ { + Name: "volume-1", DataVolume: &cdicorev1alpha1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -124,6 +148,25 @@ var _ = Describe("PluginSPIImpl", func() { }, }, }, + { + Name: "volume-2", + DataVolume: &cdicorev1alpha1.DataVolumeSpec{ + PVC: &corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + StorageClassName: pointer.StringPtr(storageClassName), + }, + Source: cdicorev1alpha1.DataVolumeSource{ + Blank: &cdicorev1alpha1.DataVolumeBlankImage{}, + }, + }, + }, }, SSHKeys: []string{ sshPublicKey, @@ -184,12 +227,13 @@ var _ = Describe("PluginSPIImpl", func() { Devices: kubevirtv1.Devices{ Disks: []kubevirtv1.Disk{ { - Name: "rootdisk", + Name: api.RootDiskName, DiskDevice: kubevirtv1.DiskDevice{ Disk: &kubevirtv1.DiskTarget{ Bus: "virtio", }, }, + DedicatedIOThread: pointer.BoolPtr(true), }, { Name: "cloudinitdisk", @@ -206,8 +250,19 @@ var _ = Describe("PluginSPIImpl", func() { Bus: "virtio", }, }, + DedicatedIOThread: pointer.BoolPtr(true), + }, + { + Name: "disk1", + DiskDevice: kubevirtv1.DiskDevice{ + Disk: &kubevirtv1.DiskTarget{ + Bus: "virtio", + }, + }, }, }, + BlockMultiQueue: pointer.BoolPtr(true), + NetworkInterfaceMultiQueue: pointer.BoolPtr(false), Interfaces: []kubevirtv1.Interface{ { Name: "default", @@ -249,7 +304,7 @@ var _ = Describe("PluginSPIImpl", func() { TerminationGracePeriodSeconds: pointer.Int64Ptr(30), Volumes: []kubevirtv1.Volume{ { - Name: "rootdisk", + Name: api.RootDiskName, VolumeSource: kubevirtv1.VolumeSource{ DataVolume: &kubevirtv1.DataVolumeSource{ Name: machineName, @@ -281,6 +336,14 @@ ethernets: }, }, }, + { + Name: "disk1", + VolumeSource: kubevirtv1.VolumeSource{ + DataVolume: &kubevirtv1.DataVolumeSource{ + Name: machineName + "-1", + }, + }, + }, }, Networks: []kubevirtv1.Network{ { @@ -317,6 +380,13 @@ ethernets: }, Spec: *providerSpec.AdditionalVolumes[0].DataVolume, }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: machineName + "-1", + Namespace: namespace, + }, + Spec: *providerSpec.AdditionalVolumes[1].DataVolume, + }, }, }, } diff --git a/pkg/kubevirt/core/util.go b/pkg/kubevirt/core/util.go index 0d78911..9113af9 100644 --- a/pkg/kubevirt/core/util.go +++ b/pkg/kubevirt/core/util.go @@ -167,22 +167,23 @@ func buildVolumes( machineName, namespace, userDataSecretName, networkData string, rootVolume cdicorev1alpha1.DataVolumeSpec, additionalVolumes []api.AdditionalVolumeSpec, + configuredDisks []kubevirtv1.Disk, ) ([]kubevirtv1.Disk, []kubevirtv1.Volume, []cdicorev1alpha1.DataVolume) { var disks []kubevirtv1.Disk var volumes []kubevirtv1.Volume var dataVolumes []cdicorev1alpha1.DataVolume // Append a disk, a volume, and a data volume for the root disk - disks = append(disks, kubevirtv1.Disk{ - Name: "rootdisk", - DiskDevice: kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: "virtio", - }, - }, - }) + var rootDisk kubevirtv1.Disk + if d := findDiskByName(api.RootDiskName, configuredDisks); d != nil { + rootDisk = *d + } else { + rootDisk = buildDefaultDisk(api.RootDiskName) + } + + disks = append(disks, rootDisk) volumes = append(volumes, kubevirtv1.Volume{ - Name: "rootdisk", + Name: api.RootDiskName, VolumeSource: kubevirtv1.VolumeSource{ DataVolume: &kubevirtv1.DataVolumeSource{ Name: machineName, @@ -223,15 +224,14 @@ func buildVolumes( // Generate a unique name for this disk diskName := fmt.Sprintf("disk%d", i) - // Append a disk for this additional disk - disks = append(disks, kubevirtv1.Disk{ - Name: diskName, - DiskDevice: kubevirtv1.DiskDevice{ - Disk: &kubevirtv1.DiskTarget{ - Bus: "virtio", - }, - }, - }) + var disk kubevirtv1.Disk + if d := findDiskByName(volume.Name, configuredDisks); d != nil { + disk = *d + disk.Name = diskName + } else { + disk = buildDefaultDisk(diskName) + } + disks = append(disks, disk) switch { case volume.DataVolume != nil: @@ -271,6 +271,26 @@ func buildVolumes( return disks, volumes, dataVolumes } +func findDiskByName(name string, disks []kubevirtv1.Disk) *kubevirtv1.Disk { + for _, disk := range disks { + if name == disk.Name { + return &disk + } + } + return nil +} + +func buildDefaultDisk(name string) kubevirtv1.Disk { + return kubevirtv1.Disk{ + Name: name, + DiskDevice: kubevirtv1.DiskDevice{ + Disk: &kubevirtv1.DiskTarget{ + Bus: "virtio", + }, + }, + } +} + const ( // defaultRegion is the name of the default region. // VMs using this region are scheduled on nodes for which a region failure domain is not specified. diff --git a/pkg/kubevirt/validation/validation.go b/pkg/kubevirt/validation/validation.go index 33f3267..61daaa4 100644 --- a/pkg/kubevirt/validation/validation.go +++ b/pkg/kubevirt/validation/validation.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/tools/clientcmd" cdicorev1alpha1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" @@ -51,6 +52,10 @@ func ValidateKubevirtProviderSpec(spec *api.KubeVirtProviderSpec) field.ErrorLis for i, volume := range spec.AdditionalVolumes { volumePath := field.NewPath("additionalVolumes").Index(i) + if volume.Name == "" { + errs = append(errs, field.Required(volumePath.Child("name"), "cannot be empty")) + } + switch { case volume.DataVolume != nil: errs = append(errs, validateDataVolume(volumePath.Child("dataVolume"), volume.DataVolume)...) @@ -81,9 +86,46 @@ func ValidateKubevirtProviderSpec(spec *api.KubeVirtProviderSpec) field.ErrorLis } } + if spec.Devices != nil { + disksPath := field.NewPath("devices").Child("disks") + disks := sets.NewString() + + // +1 because of root-disk which is required and unique + volumesLen := len(spec.AdditionalVolumes) + 1 + + if disksLen := len(spec.Devices.Disks); disksLen > volumesLen { + errs = append(errs, field.Invalid(disksPath, disksLen, "the number of disks is larger than the number of volumes")) + } + + for i, disk := range spec.Devices.Disks { + if disk.BootOrder != nil { + errs = append(errs, field.Forbidden(disksPath.Index(i).Child("bootOrder"), "cannot be set")) + } + + if disk.Name == "" { + errs = append(errs, field.Required(disksPath.Index(i).Child("name"), "cannot be empty")) + } else if disks.Has(disk.Name) { + errs = append(errs, field.Invalid(disksPath.Index(i).Child("name"), disk.Name, "already exists")) + continue + } else if !hasVolumeWithName(disk.Name, spec.AdditionalVolumes) && disk.Name != api.RootDiskName { + errs = append(errs, field.Invalid(disksPath.Index(i).Child("name"), disk.Name, "no matching volume")) + } + disks.Insert(disk.Name) + } + } + return errs } +func hasVolumeWithName(diskName string, volumes []api.AdditionalVolumeSpec) bool { + for _, volume := range volumes { + if volume.Name == diskName { + return true + } + } + return false +} + // ValidateKubevirtProviderSecret validates the given kubevirt provider secret. func ValidateKubevirtProviderSecret(secret *corev1.Secret) field.ErrorList { errs := field.ErrorList{}