[OSPK8-430] Introduce additional disk support with v1beta2

Allows adding additional disks to a vm role, therefore
restructures disk specification for vmset/ctlplane VM roles.
Therefore introduces new API verstion for VMSet and Ctlplane.
Adds OpenStackVMSetDisk type which describes any disk
configured for the VM.
Root of vmset now has RootDisk of type OpenStackVMSetDisk
and AdditionalDisks of type []OpenStackVMSetDisk.
OpenStackVMSetDisk with the parameters:
- Name
- DiskSize
- StorageClass
- StorageAccessMode
- StorageVolumeMode
- DedicatedIOThread

* Also adds IOThreadsPolicy and BlockMultiQueue to the root vmset spec:

DedicatedIOThread - Disks with dedicatedIOThread set to true will be
allocated an exclusive thread. This is generally useful if a specific
Disk is expected to have heavy I/O traffic, e.g. a database spindle.

If IOThreadsPolicy is default, use of IOThreads will be disabled. However,
if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled
and default to shared.
When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from
the vCPUs and placed on the same physical CPU as the QEMU emulator thread.
An ioThreadsPolicy of shared indicates that KubeVirt should use one thread
that will be shared by all disk devices.

Block Multi-Queue is a framework for the Linux block layer that maps Device
I/O queries to multiple queues.
This splits I/O processing up across multiple threads, and therefor multiple
CPUs. libvirt recommends that the number of queues used should match the
number of CPUs allocated for optimal performance.

* API update

For existing CRs via the ctlplane mutation webhook parameter from
old specification gets moved into the new structure. When done
an annotation gets set on the object to no longer check it.

To auto trigger the storage verstion migration the ctlplane controller
reconcilation creates the required StorageVersionMigration via
ensureStorageVersionMigration(). The ctlplane controller then updates/
patches the owning VMSet objects.

* Validations vi webhook

On CR create:
additional disks checks
- validate Disk Name is not 'rootdisk' as this is reserved for the boot disk
- validate that disk name is uniq within the VM role

On CR update:

rootdisk checks
- validate DiskSize don't change
- validate StorageAccessMode don't change
- validate StorageClass don't change
- validate StorageVolumeMode don't change

additional disks checks
- same as rootdisk update checks
- validate Disk Name is not 'rootdisk' as this is reserved for the boot disk
- validate that disk name is uniq within the VM role

PROJECT

@@ -184,4 +184,28 @@ resources:
     defaulting: true
     validation: true
     webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: openstack.org
+  group: osp-director
+  kind: OpenStackVMSet
+  path: github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2
+  version: v1beta2
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: openstack.org
+  group: osp-director
+  kind: OpenStackControlPlane
+  path: github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2
+  version: v1beta2
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 version: "3"

README.md

@@ -483,16 +483,38 @@ Create a base RHEL data volume prior to deploying OpenStack.  This will be used
             - storagemgmt
           cores: 6
           memory: 12
-          diskSize: 50
-          baseImageVolumeName: openstack-base-img
-          # storageClass must support RWX to be able to live migrate VMs
-          storageClass: host-nfs-storageclass
-          storageAccessMode: ReadWriteMany
-          # When using OpenShift Virtualization with OpenShift Container Platform Container Storage,
-          # specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks,
-          # RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs.
-          # To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block.
-          storageVolumeMode: Filesystem
+          rootDisk:
+            diskSize: 50
+            baseImageVolumeName: openstack-base-img
+            # storageClass must support RWX to be able to live migrate VMs
+            storageClass: host-nfs-storageclass
+            storageAccessMode: ReadWriteMany
+            # When using OpenShift Virtualization with OpenShift Container Platform Container Storage,
+            # specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks,
+            # RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs.
+            # To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block.
+            storageVolumeMode: Filesystem
+            # Optional
+            # DedicatedIOThread - Disks with dedicatedIOThread set to true will be allocated an exclusive thread.
+            # This is generally useful if a specific Disk is expected to have heavy I/O traffic, e.g. a database spindle.
+            dedicatedIOThread: false
+          additionalDisks:
+            # name must be uniqe and must not be rootDisk
+            - name: dataDisk1
+              diskSize: 100
+              storageClass: host-nfs-storageclass
+              storageAccessMode: ReadWriteMany
+              storageVolumeMode: Filesystem
+          # Optional block storage settings
+          # IOThreadsPolicy - IO thread policy for the domain. Currently valid policies are shared and auto.
+          # However, if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled and default to shared.
+          # When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from the vCPUs and placed on the same physical CPU as the QEMU emulator thread.
+          # An ioThreadsPolicy of shared indicates that KubeVirt should use one thread that will be shared by all disk devices.
+          ioThreadsPolicy: auto
+          # Block Multi-Queue is a framework for the Linux block layer that maps Device I/O queries to multiple queues.
+          # This splits I/O processing up across multiple threads, and therefor multiple CPUs. libvirt recommends that the
+          # number of queues used should match the number of CPUs allocated for optimal performance.
+          blockMultiQueue: false
     ```
 
     If you write the above YAML into a file called openstackcontrolplane.yaml you can create the OpenStackControlPlane via this command:
@@ -1334,9 +1356,12 @@ spec:
         - storage_mgmt
       cores: 6
       memory: 20
-      diskSize: 40
-      baseImageVolumeName: controller-base-img
-      storageClass: host-nfs-storageclass
+      rootDisk:
+        diskSize: 40
+        baseImageVolumeName: controller-base-img
+        storageClass: host-nfs-storageclass
+        storageAccessMode: ReadWriteMany
+        storageVolumeMode: Filesystem
   enableFencing: False
 ```
 

api/shared/common_types.go

@@ -28,18 +28,18 @@ const (
 	APIActionDelete APIAction = "delete"
 )
 
-// Hash - struct to add hashes to status
-type Hash struct {
-	// Name of hash referencing the parameter
-	Name string `json:"name,omitempty"`
-	// Hash
-	Hash string `json:"hash,omitempty"`
-}
-
 // ProvisioningState - the overall state of all VMs in this OpenStackVmSet
 type ProvisioningState string
 
 const (
 	// HostRefInitState - intial HostRef state of a new node which has not yet assigned
 	HostRefInitState string = "unassigned"
+
+	// RootDiskConvertedAnnotation - annotation set on object when root disk definition got migrated
+	RootDiskConvertedAnnotation string = "osp-director.openstack.org/rootdiskconverted"
 )
+
+// OpenStackControlPlaneDefaults -
+type OpenStackControlPlaneDefaults struct {
+	OpenStackRelease string
+}

api/shared/conditions.go

@@ -588,6 +588,8 @@ const (
 	VMSetCondReasonNamespaceFencingDataError ConditionReason = "NamespaceFencingDataError"
 	// VMSetCondReasonKubevirtFencingServiceAccountError - error creating/reading the KubevirtFencingServiceAccount secret
 	VMSetCondReasonKubevirtFencingServiceAccountError ConditionReason = "KubevirtFencingServiceAccountError"
+	// VMSetCondReasonKubevirtError - error creating/updating Kubevirt object
+	VMSetCondReasonKubevirtError ConditionReason = "KubevirtError"
 	// VMSetCondReasonKubeConfigError - error getting the KubeConfig used by the operator
 	VMSetCondReasonKubeConfigError ConditionReason = "KubeConfigError"
 	// VMSetCondReasonCloudInitSecretError - error creating the CloudInitSecret

api/v1beta1/common_backup.go

@@ -109,7 +109,8 @@ func GetOpenStackBackupsWithLabel(client goClient.Client, namespace string, labe
 	return osBackupList, nil
 }
 
-func checkBackupOperationBlocksAction(namespace string, action shared.APIAction) error {
+// CheckBackupOperationBlocksAction -
+func CheckBackupOperationBlocksAction(namespace string, action shared.APIAction) error {
 	op, err := GetOpenStackBackupOperationInProgress(webhookClient, namespace)
 
 	if err != nil {

api/v1beta1/common_openstacknet.go

@@ -229,8 +229,8 @@ func GetOsNetCfg(
 	return osNetCfg, nil
 }
 
-// validateNetworks - validate that for all configured subnets an osnet exists
-func validateNetworks(namespace string, networks []string) error {
+// ValidateNetworks - validate that for all configured subnets an osnet exists
+func ValidateNetworks(namespace string, networks []string) error {
 	for _, subnetName := range networks {
 		//
 		// Get OSnet with SubNetNameLabelSelector: subnetName

api/v1beta1/common_types.go

@@ -28,11 +28,6 @@ type Hash struct {
 	Hash string `json:"hash,omitempty"`
 }
 
-const (
-	// HostRefInitState - intial HostRef state of a new node which has not yet assigned
-	HostRefInitState string = "unassigned"
-)
-
 // HostStatus represents the hostname and IP info for a specific host
 type HostStatus struct {
 	Hostname          string                   `json:"hostname"`
@@ -53,20 +48,6 @@ type HostStatus struct {
 	CtlplaneIP            string `json:"ctlplaneIP"`
 }
 
-// NetworkStatus represents the network details of a network
-type NetworkStatus struct {
-	Cidr string `json:"cidr"`
-
-	// +kubebuilder:validation:Optional
-	Vlan int `json:"vlan"`
-
-	AllocationStart string `json:"allocationStart"`
-	AllocationEnd   string `json:"allocationEnd"`
-
-	// +kubebuilder:validation:Optional
-	Gateway string `json:"gateway"`
-}
-
 //
 // SyncIPsetStatus - sync relevant information from IPSet to CR status
 //
@@ -91,7 +72,7 @@ func SyncIPsetStatus(
 		hostStatus.IPAddresses = ipsetHostStatus.IPAddresses
 		hostStatus.ProvisioningState = ipsetHostStatus.ProvisioningState
 
-		if ipsetHostStatus.HostRef != HostRefInitState {
+		if ipsetHostStatus.HostRef != shared.HostRefInitState {
 			hostStatus.HostRef = ipsetHostStatus.HostRef
 		}
 	}

api/v1beta1/openstackbaremetalset_webhook.go

@@ -54,7 +54,7 @@ var _ webhook.Validator = &OpenStackBaremetalSet{}
 func (r *OpenStackBaremetalSet) ValidateCreate() error {
 	baremetalsetlog.Info("validate create", "name", r.Name)
 
-	if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil {
+	if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil {
 		return err
 	}
 
@@ -72,7 +72,7 @@ func (r *OpenStackBaremetalSet) ValidateCreate() error {
 	//
 	// validate that for all configured subnets an osnet exists
 	//
-	if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
+	if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
 		return err
 	}
 
@@ -86,7 +86,7 @@ func (r *OpenStackBaremetalSet) ValidateUpdate(old runtime.Object) error {
 	//
 	// validate that for all configured subnets an osnet exists
 	//
-	if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
+	if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
 		return err
 	}
 
@@ -98,7 +98,7 @@ func (r *OpenStackBaremetalSet) ValidateUpdate(old runtime.Object) error {
 func (r *OpenStackBaremetalSet) ValidateDelete() error {
 	baremetalsetlog.Info("validate delete", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
 }
 
 //+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackbaremetalset,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackbaremetalsets,verbs=create;update,versions=v1beta1,name=mopenstackbaremetalset.kb.io,admissionReviewVersions=v1

api/v1beta1/openstackclient_webhook.go

@@ -117,7 +117,7 @@ func (r *OpenStackClient) ValidateCreate() error {
 	//
 	// validate that for all configured subnets an osnet exists
 	//
-	if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
+	if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
 		return err
 	}
 
@@ -131,7 +131,7 @@ func (r *OpenStackClient) ValidateUpdate(old runtime.Object) error {
 	//
 	// validate that for all configured subnets an osnet exists
 	//
-	if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
+	if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil {
 		return err
 	}
 

api/v1beta1/openstackcontrolplane_types.go

@@ -77,13 +77,11 @@ type OpenStackVirtualMachineRoleSpec struct {
 	// StorageClass to be used for the controller disks
 	StorageClass string `json:"storageClass,omitempty"`
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default=ReadWriteMany
 	// +kubebuilder:validation:Enum=ReadWriteOnce;ReadWriteMany
 	// StorageAccessMode - Virtual machines must have a persistent volume claim (PVC)
 	// with a shared ReadWriteMany (RWX) access mode to be live migrated.
 	StorageAccessMode string `json:"storageAccessMode,omitempty"`
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default=Filesystem
 	// +kubebuilder:validation:Enum=Block;Filesystem
 	// StorageVolumeMode - When using OpenShift Virtualization with OpenShift Container Platform Container Storage,
 	// specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks,

api/v1beta1/openstackcontrolplane_webhook.go

@@ -34,18 +34,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
-// OpenStackControlPlaneDefaults -
-type OpenStackControlPlaneDefaults struct {
-	OpenStackRelease string
-}
-
-var openstackControlPlaneDefaults OpenStackControlPlaneDefaults
+var openstackControlPlaneDefaults shared.OpenStackControlPlaneDefaults
 
 // log is for logging in this package.
 var controlplanelog = logf.Log.WithName("controlplane-resource")
 
 // SetupWebhookWithManager - register this webhook with the controller manager
-func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaults OpenStackControlPlaneDefaults) error {
+func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaults shared.OpenStackControlPlaneDefaults) error {
 
 	openstackControlPlaneDefaults = defaults
 
@@ -58,15 +53,15 @@ func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaul
 		Complete()
 }
 
-// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,versions=v1beta1,name=vopenstackcontrolplane.kb.io,admissionReviewVersions=v1
+//// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,versions=v1beta1,name=vopenstackcontrolplanev1beta1.kb.io,admissionReviewVersions=v1
 
 var _ webhook.Validator = &OpenStackControlPlane{}
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type
 func (r *OpenStackControlPlane) ValidateCreate() error {
 	controlplanelog.Info("validate create", "name", r.Name)
 
-	if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil {
+	if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil {
 		return err
 	}
 
@@ -112,7 +107,7 @@ func (r *OpenStackControlPlane) ValidateCreate() error {
 		//
 		// validate that for all configured subnets an osnet exists
 		//
-		if err := validateNetworks(r.GetNamespace(), vmspec.Networks); err != nil {
+		if err := ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil {
 			return err
 		}
 	}
@@ -141,7 +136,7 @@ func (r *OpenStackControlPlane) ValidateUpdate(old runtime.Object) error {
 		//
 		// validate that for all configured subnets an osnet exists
 		//
-		if err := validateNetworks(r.GetNamespace(), vmspec.Networks); err != nil {
+		if err := ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil {
 			return err
 		}
 	}
@@ -153,10 +148,10 @@ func (r *OpenStackControlPlane) ValidateUpdate(old runtime.Object) error {
 func (r *OpenStackControlPlane) ValidateDelete() error {
 	controlplanelog.Info("validate delete", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
 }
 
-//+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update,versions=v1beta1,name=mopenstackcontrolplane.kb.io,admissionReviewVersions=v1
+////+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update,versions=v1beta1,name=mopenstackcontrolplanev1beta1.kb.io,admissionReviewVersions=v1
 
 var _ webhook.Defaulter = &OpenStackControlPlane{}
 

api/v1beta1/openstacknet_webhook.go

@@ -80,7 +80,7 @@ var _ webhook.Validator = &OpenStackNet{}
 func (r *OpenStackNet) ValidateCreate() error {
 	openstacknetlog.Info("validate create", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate)
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
@@ -130,5 +130,5 @@ func (r *OpenStackNet) ValidateUpdate(old runtime.Object) error {
 func (r *OpenStackNet) ValidateDelete() error {
 	openstacknetlog.Info("validate delete", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
 }

api/v1beta1/openstacknetattachment_webhook.go

@@ -67,7 +67,7 @@ var _ webhook.Validator = &OpenStackNetAttachment{}
 func (r *OpenStackNetAttachment) ValidateCreate() error {
 	openstacknetattachmentlog.Info("validate create", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate)
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
@@ -81,7 +81,7 @@ func (r *OpenStackNetAttachment) ValidateUpdate(old runtime.Object) error {
 func (r *OpenStackNetAttachment) ValidateDelete() error {
 	openstacknetattachmentlog.Info("validate delete", "name", r.Name)
 
-	return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
+	return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete)
 }
 
 func (r *OpenStackNetAttachment) checkBridgeName(old runtime.Object) error {