From dbcc3d1a1d780382cdac5036c3bddcc9522c680a Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Wed, 13 Apr 2022 14:07:20 +0000 Subject: [PATCH] [OSPK8-430] Introduce additional disk support with v1beta2 Allows adding additional disks to a vm role, therefore restructures disk specification for vmset/ctlplane VM roles. Therefore introduces new API verstion for VMSet and Ctlplane. Adds OpenStackVMSetDisk type which describes any disk configured for the VM. Root of vmset now has RootDisk of type OpenStackVMSetDisk and AdditionalDisks of type []OpenStackVMSetDisk. OpenStackVMSetDisk with the parameters: - Name - DiskSize - StorageClass - StorageAccessMode - StorageVolumeMode - DedicatedIOThread * Also adds IOThreadsPolicy and BlockMultiQueue to the root vmset spec: DedicatedIOThread - Disks with dedicatedIOThread set to true will be allocated an exclusive thread. This is generally useful if a specific Disk is expected to have heavy I/O traffic, e.g. a database spindle. If IOThreadsPolicy is default, use of IOThreads will be disabled. However, if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled and default to shared. When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from the vCPUs and placed on the same physical CPU as the QEMU emulator thread. An ioThreadsPolicy of shared indicates that KubeVirt should use one thread that will be shared by all disk devices. Block Multi-Queue is a framework for the Linux block layer that maps Device I/O queries to multiple queues. This splits I/O processing up across multiple threads, and therefor multiple CPUs. libvirt recommends that the number of queues used should match the number of CPUs allocated for optimal performance. * API update For existing CRs via the ctlplane mutation webhook parameter from old specification gets moved into the new structure. When done an annotation gets set on the object to no longer check it. To auto trigger the storage verstion migration the ctlplane controller reconcilation creates the required StorageVersionMigration via ensureStorageVersionMigration(). The ctlplane controller then updates/ patches the owning VMSet objects. * Validations vi webhook On CR create: additional disks checks - validate Disk Name is not 'rootdisk' as this is reserved for the boot disk - validate that disk name is uniq within the VM role On CR update: rootdisk checks - validate DiskSize don't change - validate StorageAccessMode don't change - validate StorageClass don't change - validate StorageVolumeMode don't change additional disks checks - same as rootdisk update checks - validate Disk Name is not 'rootdisk' as this is reserved for the boot disk - validate that disk name is uniq within the VM role --- PROJECT | 24 + README.md | 51 ++- api/shared/common_types.go | 16 +- api/shared/conditions.go | 2 + api/shared/zz_generated.deepcopy.go | 8 +- api/v1beta1/common_backup.go | 3 +- api/v1beta1/common_openstacknet.go | 4 +- api/v1beta1/common_types.go | 21 +- api/v1beta1/openstackbaremetalset_webhook.go | 8 +- api/v1beta1/openstackclient_webhook.go | 4 +- api/v1beta1/openstackcontrolplane_types.go | 2 - api/v1beta1/openstackcontrolplane_webhook.go | 21 +- api/v1beta1/openstacknet_webhook.go | 4 +- api/v1beta1/openstacknetattachment_webhook.go | 4 +- api/v1beta1/openstacknetconfig_webhook.go | 4 +- .../openstackprovisionserver_webhook.go | 4 +- api/v1beta1/openstackvmset_webhook.go | 12 +- api/v1beta1/zz_generated.deepcopy.go | 30 -- api/v1beta2/common_openstackcontrolplane.go | 90 ++++ api/v1beta2/common_types.go | 84 ++++ api/v1beta2/common_webhook.go | 244 ++++++++++ api/v1beta2/groupversion_info.go | 36 ++ api/v1beta2/openstackcontrolplane_types.go | 200 ++++++++ api/v1beta2/openstackcontrolplane_webhook.go | 325 +++++++++++++ api/v1beta2/openstackvmset_types.go | 190 ++++++++ api/v1beta2/openstackvmset_webhook.go | 209 +++++++++ api/v1beta2/webhook_suite_test.go | 129 ++++++ api/v1beta2/zz_generated.deepcopy.go | 418 +++++++++++++++++ ...rector.openstack.org_openstackbackups.yaml | 2 - ....openstack.org_openstackcontrolplanes.yaml | 427 ++++++++++++++++- ...irector.openstack.org_openstackvmsets.yaml | 346 ++++++++++++++ ...rector-operator.clusterserviceversion.yaml | 12 + .../openstackcontrolplane_editor_role.yaml | 24 + .../openstackcontrolplane_viewer_role.yaml | 20 + config/rbac/role.yaml | 12 + config/samples/kustomization.yaml | 2 + ...irector_v1beta2_openstackcontrolplane.yaml | 43 ++ .../osp-director_v1beta2_openstackvmset.yaml | 32 ++ config/webhook/kustomization.yaml | 2 +- config/webhook/manifests.yaml | 90 ++-- .../openstackbackuprequest_controller.go | 5 +- .../openstackbaremetalset_controller.go | 2 +- .../openstackconfiggenerator_controller.go | 9 +- .../openstackcontrolplane_controller.go | 164 +++++-- controllers/openstackdeploy_controller.go | 3 +- controllers/openstackipset_controller.go | 2 +- controllers/openstacknetconfig_controller.go | 3 +- controllers/openstackvmset_controller.go | 428 ++++++++++-------- go.mod | 10 +- go.sum | 45 +- main.go | 33 +- pkg/common/hostname.go | 6 +- pkg/openstackbackup/funcs.go | 13 +- pkg/openstackconfiggenerator/configmap.go | 6 +- pkg/vmset/virtualmachine.go | 191 ++++++++ 55 files changed, 3686 insertions(+), 393 deletions(-) create mode 100644 api/v1beta2/common_openstackcontrolplane.go create mode 100644 api/v1beta2/common_types.go create mode 100644 api/v1beta2/common_webhook.go create mode 100644 api/v1beta2/groupversion_info.go create mode 100644 api/v1beta2/openstackcontrolplane_types.go create mode 100644 api/v1beta2/openstackcontrolplane_webhook.go create mode 100644 api/v1beta2/openstackvmset_types.go create mode 100644 api/v1beta2/openstackvmset_webhook.go create mode 100644 api/v1beta2/webhook_suite_test.go create mode 100644 api/v1beta2/zz_generated.deepcopy.go create mode 100644 config/rbac/openstackcontrolplane_editor_role.yaml create mode 100644 config/rbac/openstackcontrolplane_viewer_role.yaml create mode 100644 config/samples/osp-director_v1beta2_openstackcontrolplane.yaml create mode 100644 config/samples/osp-director_v1beta2_openstackvmset.yaml diff --git a/PROJECT b/PROJECT index 01fb5693..53acce89 100644 --- a/PROJECT +++ b/PROJECT @@ -184,4 +184,28 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + domain: openstack.org + group: osp-director + kind: OpenStackVMSet + path: github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2 + version: v1beta2 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + domain: openstack.org + group: osp-director + kind: OpenStackControlPlane + path: github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2 + version: v1beta2 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/README.md b/README.md index a9d5eb0a..e6cff02d 100644 --- a/README.md +++ b/README.md @@ -483,16 +483,38 @@ Create a base RHEL data volume prior to deploying OpenStack. This will be used - storagemgmt cores: 6 memory: 12 - diskSize: 50 - baseImageVolumeName: openstack-base-img - # storageClass must support RWX to be able to live migrate VMs - storageClass: host-nfs-storageclass - storageAccessMode: ReadWriteMany - # When using OpenShift Virtualization with OpenShift Container Platform Container Storage, - # specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks, - # RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs. - # To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block. - storageVolumeMode: Filesystem + rootDisk: + diskSize: 50 + baseImageVolumeName: openstack-base-img + # storageClass must support RWX to be able to live migrate VMs + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + # When using OpenShift Virtualization with OpenShift Container Platform Container Storage, + # specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks, + # RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs. + # To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block. + storageVolumeMode: Filesystem + # Optional + # DedicatedIOThread - Disks with dedicatedIOThread set to true will be allocated an exclusive thread. + # This is generally useful if a specific Disk is expected to have heavy I/O traffic, e.g. a database spindle. + dedicatedIOThread: false + additionalDisks: + # name must be uniqe and must not be rootDisk + - name: dataDisk1 + diskSize: 100 + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + # Optional block storage settings + # IOThreadsPolicy - IO thread policy for the domain. Currently valid policies are shared and auto. + # However, if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled and default to shared. + # When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from the vCPUs and placed on the same physical CPU as the QEMU emulator thread. + # An ioThreadsPolicy of shared indicates that KubeVirt should use one thread that will be shared by all disk devices. + ioThreadsPolicy: auto + # Block Multi-Queue is a framework for the Linux block layer that maps Device I/O queries to multiple queues. + # This splits I/O processing up across multiple threads, and therefor multiple CPUs. libvirt recommends that the + # number of queues used should match the number of CPUs allocated for optimal performance. + blockMultiQueue: false ``` If you write the above YAML into a file called openstackcontrolplane.yaml you can create the OpenStackControlPlane via this command: @@ -1334,9 +1356,12 @@ spec: - storage_mgmt cores: 6 memory: 20 - diskSize: 40 - baseImageVolumeName: controller-base-img - storageClass: host-nfs-storageclass + rootDisk: + diskSize: 40 + baseImageVolumeName: controller-base-img + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem enableFencing: False ``` diff --git a/api/shared/common_types.go b/api/shared/common_types.go index f75cac68..384b1e4d 100644 --- a/api/shared/common_types.go +++ b/api/shared/common_types.go @@ -28,18 +28,18 @@ const ( APIActionDelete APIAction = "delete" ) -// Hash - struct to add hashes to status -type Hash struct { - // Name of hash referencing the parameter - Name string `json:"name,omitempty"` - // Hash - Hash string `json:"hash,omitempty"` -} - // ProvisioningState - the overall state of all VMs in this OpenStackVmSet type ProvisioningState string const ( // HostRefInitState - intial HostRef state of a new node which has not yet assigned HostRefInitState string = "unassigned" + + // RootDiskConvertedAnnotation - annotation set on object when root disk definition got migrated + RootDiskConvertedAnnotation string = "osp-director.openstack.org/rootdiskconverted" ) + +// OpenStackControlPlaneDefaults - +type OpenStackControlPlaneDefaults struct { + OpenStackRelease string +} diff --git a/api/shared/conditions.go b/api/shared/conditions.go index 02baca63..1ca1b024 100644 --- a/api/shared/conditions.go +++ b/api/shared/conditions.go @@ -588,6 +588,8 @@ const ( VMSetCondReasonNamespaceFencingDataError ConditionReason = "NamespaceFencingDataError" // VMSetCondReasonKubevirtFencingServiceAccountError - error creating/reading the KubevirtFencingServiceAccount secret VMSetCondReasonKubevirtFencingServiceAccountError ConditionReason = "KubevirtFencingServiceAccountError" + // VMSetCondReasonKubevirtError - error creating/updating Kubevirt object + VMSetCondReasonKubevirtError ConditionReason = "KubevirtError" // VMSetCondReasonKubeConfigError - error getting the KubeConfig used by the operator VMSetCondReasonKubeConfigError ConditionReason = "KubeConfigError" // VMSetCondReasonCloudInitSecretError - error creating the CloudInitSecret diff --git a/api/shared/zz_generated.deepcopy.go b/api/shared/zz_generated.deepcopy.go index f363031c..db95a6d1 100644 --- a/api/shared/zz_generated.deepcopy.go +++ b/api/shared/zz_generated.deepcopy.go @@ -77,16 +77,16 @@ func (in ConditionList) DeepCopy() ConditionList { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Hash) DeepCopyInto(out *Hash) { +func (in *OpenStackControlPlaneDefaults) DeepCopyInto(out *OpenStackControlPlaneDefaults) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Hash. -func (in *Hash) DeepCopy() *Hash { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneDefaults. +func (in *OpenStackControlPlaneDefaults) DeepCopy() *OpenStackControlPlaneDefaults { if in == nil { return nil } - out := new(Hash) + out := new(OpenStackControlPlaneDefaults) in.DeepCopyInto(out) return out } diff --git a/api/v1beta1/common_backup.go b/api/v1beta1/common_backup.go index a7e4c481..1cb67dbb 100644 --- a/api/v1beta1/common_backup.go +++ b/api/v1beta1/common_backup.go @@ -109,7 +109,8 @@ func GetOpenStackBackupsWithLabel(client goClient.Client, namespace string, labe return osBackupList, nil } -func checkBackupOperationBlocksAction(namespace string, action shared.APIAction) error { +// CheckBackupOperationBlocksAction - +func CheckBackupOperationBlocksAction(namespace string, action shared.APIAction) error { op, err := GetOpenStackBackupOperationInProgress(webhookClient, namespace) if err != nil { diff --git a/api/v1beta1/common_openstacknet.go b/api/v1beta1/common_openstacknet.go index d4905ca0..98952325 100644 --- a/api/v1beta1/common_openstacknet.go +++ b/api/v1beta1/common_openstacknet.go @@ -229,8 +229,8 @@ func GetOsNetCfg( return osNetCfg, nil } -// validateNetworks - validate that for all configured subnets an osnet exists -func validateNetworks(namespace string, networks []string) error { +// ValidateNetworks - validate that for all configured subnets an osnet exists +func ValidateNetworks(namespace string, networks []string) error { for _, subnetName := range networks { // // Get OSnet with SubNetNameLabelSelector: subnetName diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index e6e3bf3d..f59c97a9 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -28,11 +28,6 @@ type Hash struct { Hash string `json:"hash,omitempty"` } -const ( - // HostRefInitState - intial HostRef state of a new node which has not yet assigned - HostRefInitState string = "unassigned" -) - // HostStatus represents the hostname and IP info for a specific host type HostStatus struct { Hostname string `json:"hostname"` @@ -53,20 +48,6 @@ type HostStatus struct { CtlplaneIP string `json:"ctlplaneIP"` } -// NetworkStatus represents the network details of a network -type NetworkStatus struct { - Cidr string `json:"cidr"` - - // +kubebuilder:validation:Optional - Vlan int `json:"vlan"` - - AllocationStart string `json:"allocationStart"` - AllocationEnd string `json:"allocationEnd"` - - // +kubebuilder:validation:Optional - Gateway string `json:"gateway"` -} - // // SyncIPsetStatus - sync relevant information from IPSet to CR status // @@ -91,7 +72,7 @@ func SyncIPsetStatus( hostStatus.IPAddresses = ipsetHostStatus.IPAddresses hostStatus.ProvisioningState = ipsetHostStatus.ProvisioningState - if ipsetHostStatus.HostRef != HostRefInitState { + if ipsetHostStatus.HostRef != shared.HostRefInitState { hostStatus.HostRef = ipsetHostStatus.HostRef } } diff --git a/api/v1beta1/openstackbaremetalset_webhook.go b/api/v1beta1/openstackbaremetalset_webhook.go index 5e4fed64..50805aad 100644 --- a/api/v1beta1/openstackbaremetalset_webhook.go +++ b/api/v1beta1/openstackbaremetalset_webhook.go @@ -54,7 +54,7 @@ var _ webhook.Validator = &OpenStackBaremetalSet{} func (r *OpenStackBaremetalSet) ValidateCreate() error { baremetalsetlog.Info("validate create", "name", r.Name) - if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { return err } @@ -72,7 +72,7 @@ func (r *OpenStackBaremetalSet) ValidateCreate() error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } @@ -86,7 +86,7 @@ func (r *OpenStackBaremetalSet) ValidateUpdate(old runtime.Object) error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } @@ -98,7 +98,7 @@ func (r *OpenStackBaremetalSet) ValidateUpdate(old runtime.Object) error { func (r *OpenStackBaremetalSet) ValidateDelete() error { baremetalsetlog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } //+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackbaremetalset,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackbaremetalsets,verbs=create;update,versions=v1beta1,name=mopenstackbaremetalset.kb.io,admissionReviewVersions=v1 diff --git a/api/v1beta1/openstackclient_webhook.go b/api/v1beta1/openstackclient_webhook.go index 3acb64ae..19a342df 100644 --- a/api/v1beta1/openstackclient_webhook.go +++ b/api/v1beta1/openstackclient_webhook.go @@ -117,7 +117,7 @@ func (r *OpenStackClient) ValidateCreate() error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } @@ -131,7 +131,7 @@ func (r *OpenStackClient) ValidateUpdate(old runtime.Object) error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } diff --git a/api/v1beta1/openstackcontrolplane_types.go b/api/v1beta1/openstackcontrolplane_types.go index 2443201d..c4e00dfa 100644 --- a/api/v1beta1/openstackcontrolplane_types.go +++ b/api/v1beta1/openstackcontrolplane_types.go @@ -77,13 +77,11 @@ type OpenStackVirtualMachineRoleSpec struct { // StorageClass to be used for the controller disks StorageClass string `json:"storageClass,omitempty"` // +kubebuilder:validation:Optional - // +kubebuilder:default=ReadWriteMany // +kubebuilder:validation:Enum=ReadWriteOnce;ReadWriteMany // StorageAccessMode - Virtual machines must have a persistent volume claim (PVC) // with a shared ReadWriteMany (RWX) access mode to be live migrated. StorageAccessMode string `json:"storageAccessMode,omitempty"` // +kubebuilder:validation:Optional - // +kubebuilder:default=Filesystem // +kubebuilder:validation:Enum=Block;Filesystem // StorageVolumeMode - When using OpenShift Virtualization with OpenShift Container Platform Container Storage, // specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks, diff --git a/api/v1beta1/openstackcontrolplane_webhook.go b/api/v1beta1/openstackcontrolplane_webhook.go index a3ec1846..4dc15830 100644 --- a/api/v1beta1/openstackcontrolplane_webhook.go +++ b/api/v1beta1/openstackcontrolplane_webhook.go @@ -34,18 +34,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" ) -// OpenStackControlPlaneDefaults - -type OpenStackControlPlaneDefaults struct { - OpenStackRelease string -} - -var openstackControlPlaneDefaults OpenStackControlPlaneDefaults +var openstackControlPlaneDefaults shared.OpenStackControlPlaneDefaults // log is for logging in this package. var controlplanelog = logf.Log.WithName("controlplane-resource") // SetupWebhookWithManager - register this webhook with the controller manager -func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaults OpenStackControlPlaneDefaults) error { +func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaults shared.OpenStackControlPlaneDefaults) error { openstackControlPlaneDefaults = defaults @@ -58,7 +53,7 @@ func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaul Complete() } -// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,versions=v1beta1,name=vopenstackcontrolplane.kb.io,admissionReviewVersions=v1 +//// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,versions=v1beta1,name=vopenstackcontrolplanev1beta1.kb.io,admissionReviewVersions=v1 var _ webhook.Validator = &OpenStackControlPlane{} @@ -66,7 +61,7 @@ var _ webhook.Validator = &OpenStackControlPlane{} func (r *OpenStackControlPlane) ValidateCreate() error { controlplanelog.Info("validate create", "name", r.Name) - if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { return err } @@ -112,7 +107,7 @@ func (r *OpenStackControlPlane) ValidateCreate() error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { return err } } @@ -141,7 +136,7 @@ func (r *OpenStackControlPlane) ValidateUpdate(old runtime.Object) error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { return err } } @@ -153,10 +148,10 @@ func (r *OpenStackControlPlane) ValidateUpdate(old runtime.Object) error { func (r *OpenStackControlPlane) ValidateDelete() error { controlplanelog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } -//+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update,versions=v1beta1,name=mopenstackcontrolplane.kb.io,admissionReviewVersions=v1 +////+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update,versions=v1beta1,name=mopenstackcontrolplanev1beta1.kb.io,admissionReviewVersions=v1 var _ webhook.Defaulter = &OpenStackControlPlane{} diff --git a/api/v1beta1/openstacknet_webhook.go b/api/v1beta1/openstacknet_webhook.go index 617d8f10..daaa313d 100644 --- a/api/v1beta1/openstacknet_webhook.go +++ b/api/v1beta1/openstacknet_webhook.go @@ -80,7 +80,7 @@ var _ webhook.Validator = &OpenStackNet{} func (r *OpenStackNet) ValidateCreate() error { openstacknetlog.Info("validate create", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type @@ -130,5 +130,5 @@ func (r *OpenStackNet) ValidateUpdate(old runtime.Object) error { func (r *OpenStackNet) ValidateDelete() error { openstacknetlog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } diff --git a/api/v1beta1/openstacknetattachment_webhook.go b/api/v1beta1/openstacknetattachment_webhook.go index 8c74cb1c..a8dc6909 100644 --- a/api/v1beta1/openstacknetattachment_webhook.go +++ b/api/v1beta1/openstacknetattachment_webhook.go @@ -67,7 +67,7 @@ var _ webhook.Validator = &OpenStackNetAttachment{} func (r *OpenStackNetAttachment) ValidateCreate() error { openstacknetattachmentlog.Info("validate create", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type @@ -81,7 +81,7 @@ func (r *OpenStackNetAttachment) ValidateUpdate(old runtime.Object) error { func (r *OpenStackNetAttachment) ValidateDelete() error { openstacknetattachmentlog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } func (r *OpenStackNetAttachment) checkBridgeName(old runtime.Object) error { diff --git a/api/v1beta1/openstacknetconfig_webhook.go b/api/v1beta1/openstacknetconfig_webhook.go index 939278de..646edcf5 100644 --- a/api/v1beta1/openstacknetconfig_webhook.go +++ b/api/v1beta1/openstacknetconfig_webhook.go @@ -169,7 +169,7 @@ func (r *OpenStackNetConfig) ValidateCreate() error { return err } - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type @@ -220,7 +220,7 @@ func (r *OpenStackNetConfig) ValidateDelete() error { openstacknetconfiglog.Info("validate delete", "name", r.Name) // TODO(user): fill in your validation logic upon object deletion. - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } // validateControlPlaneNetworkNames - validate that the specified control plane network name and name_lower match the expected ooo names diff --git a/api/v1beta1/openstackprovisionserver_webhook.go b/api/v1beta1/openstackprovisionserver_webhook.go index 2f3a4863..8cb35c36 100644 --- a/api/v1beta1/openstackprovisionserver_webhook.go +++ b/api/v1beta1/openstackprovisionserver_webhook.go @@ -65,7 +65,7 @@ var _ webhook.Validator = &OpenStackProvisionServer{} func (r *OpenStackProvisionServer) ValidateCreate() error { openstackprovisionserverlog.Info("validate create", "name", r.Name) - if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { return err } @@ -98,7 +98,7 @@ func (r *OpenStackProvisionServer) validateCr() error { func (r *OpenStackProvisionServer) ValidateDelete() error { openstackprovisionserverlog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } //+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackprovisionserver,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackprovisionservers,verbs=create;update,versions=v1beta1,name=mopenstackprovisionserver.kb.io,admissionReviewVersions=v1 diff --git a/api/v1beta1/openstackvmset_webhook.go b/api/v1beta1/openstackvmset_webhook.go index 955ce348..534a79fa 100644 --- a/api/v1beta1/openstackvmset_webhook.go +++ b/api/v1beta1/openstackvmset_webhook.go @@ -46,7 +46,7 @@ func (r *OpenStackVMSet) SetupWebhookWithManager(mgr ctrl.Manager) error { Complete() } -// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackvmset,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,versions=v1beta1,name=vopenstackvmset.kb.io,admissionReviewVersions=v1 +//// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-osp-director-openstack-org-v1beta1-openstackvmset,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,versions=v1beta1,name=vopenstackvmset.kb.io,admissionReviewVersions=v1 var _ webhook.Validator = &OpenStackVMSet{} @@ -54,7 +54,7 @@ var _ webhook.Validator = &OpenStackVMSet{} func (r *OpenStackVMSet) ValidateCreate() error { vmsetlog.Info("validate create", "name", r.Name) - if err := checkBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + if err := CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { return err } @@ -72,7 +72,7 @@ func (r *OpenStackVMSet) ValidateCreate() error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } @@ -86,7 +86,7 @@ func (r *OpenStackVMSet) ValidateUpdate(old runtime.Object) error { // // validate that for all configured subnets an osnet exists // - if err := validateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + if err := ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { return err } @@ -97,10 +97,10 @@ func (r *OpenStackVMSet) ValidateUpdate(old runtime.Object) error { func (r *OpenStackVMSet) ValidateDelete() error { vmsetlog.Info("validate delete", "name", r.Name) - return checkBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) + return CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) } -//+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackvmset,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,verbs=create;update,versions=v1beta1,name=mopenstackvmset.kb.io,admissionReviewVersions=v1 +////+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta1-openstackvmset,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,verbs=create;update,versions=v1beta1,name=mopenstackvmset.kb.io,admissionReviewVersions=v1 var _ webhook.Defaulter = &OpenStackVMSet{} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index aa774c70..90dd28ea 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -317,21 +317,6 @@ func (in *Network) DeepCopy() *Network { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. -func (in *NetworkStatus) DeepCopy() *NetworkStatus { - if in == nil { - return nil - } - out := new(NetworkStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeConfigurationPolicy) DeepCopyInto(out *NodeConfigurationPolicy) { *out = *in @@ -1109,21 +1094,6 @@ func (in *OpenStackControlPlane) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OpenStackControlPlaneDefaults) DeepCopyInto(out *OpenStackControlPlaneDefaults) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneDefaults. -func (in *OpenStackControlPlaneDefaults) DeepCopy() *OpenStackControlPlaneDefaults { - if in == nil { - return nil - } - out := new(OpenStackControlPlaneDefaults) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenStackControlPlaneList) DeepCopyInto(out *OpenStackControlPlaneList) { *out = *in diff --git a/api/v1beta2/common_openstackcontrolplane.go b/api/v1beta2/common_openstackcontrolplane.go new file mode 100644 index 00000000..b980c338 --- /dev/null +++ b/api/v1beta2/common_openstackcontrolplane.go @@ -0,0 +1,90 @@ +package v1beta2 + +import ( + "context" + "fmt" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +// +// GetControlPlane - Get OSP ControlPlane CR where e.g. the status information has the +// OSP version: controlPlane.Status.OSPVersion +// FIXME: We assume there is only one ControlPlane CR for now (enforced by webhook), but this might need to change +// +func GetControlPlane( + c client.Client, + obj metav1.Object, +) (OpenStackControlPlane, reconcile.Result, error) { + + controlPlane := OpenStackControlPlane{} + controlPlaneList := &OpenStackControlPlaneList{} + controlPlaneListOpts := []client.ListOption{ + client.InNamespace(obj.GetNamespace()), + client.Limit(1000), + } + err := c.List(context.TODO(), controlPlaneList, controlPlaneListOpts...) + if err != nil { + return controlPlane, ctrl.Result{}, err + } + + if len(controlPlaneList.Items) == 0 { + err := fmt.Errorf("no OpenStackControlPlanes found in namespace %s. Requeing", obj.GetNamespace()) + return controlPlane, ctrl.Result{RequeueAfter: time.Second * 10}, err + } + + // FIXME: See FIXME above + controlPlane = controlPlaneList.Items[0] + + return controlPlane, ctrl.Result{}, nil + +} + +// CreateVIPNetworkList - return list of all networks from all VM roles which has vip flag +func CreateVIPNetworkList( + c client.Client, + instance *OpenStackControlPlane, +) ([]string, error) { + + // create uniq list networls of all VirtualMachineRoles + networkList := make(map[string]bool) + uniqNetworksList := []string{} + + for _, vmRole := range instance.Spec.VirtualMachineRoles { + for _, netNameLower := range vmRole.Networks { + // get network with name_lower label + labelSelector := map[string]string{ + shared.SubNetNameLabelSelector: netNameLower, + } + + // get network with name_lower label to verify if VIP needs to be requested from Spec + network, err := ospdirectorv1beta1.GetOpenStackNetWithLabel( + c, + instance.Namespace, + labelSelector, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + return uniqNetworksList, fmt.Errorf(fmt.Sprintf("OpenStackNet with NameLower %s not found!", netNameLower)) + } + // Error reading the object - requeue the request. + return uniqNetworksList, fmt.Errorf(fmt.Sprintf("Error getting OSNet with labelSelector %v", labelSelector)) + } + + if _, value := networkList[netNameLower]; !value && network.Spec.VIP { + networkList[netNameLower] = true + uniqNetworksList = append(uniqNetworksList, netNameLower) + } + } + } + + return uniqNetworksList, nil +} diff --git a/api/v1beta2/common_types.go b/api/v1beta2/common_types.go new file mode 100644 index 00000000..24af1b93 --- /dev/null +++ b/api/v1beta2/common_types.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 Red Hat + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" +) + +// Hash - struct to add hashes to status +type Hash struct { + // Name of hash referencing the parameter + Name string `json:"name,omitempty"` + // Hash + Hash string `json:"hash,omitempty"` +} + +// HostStatus represents the hostname and IP info for a specific host +type HostStatus struct { + Hostname string `json:"hostname"` + ProvisioningState shared.ProvisioningState `json:"provisioningState"` + + // +kubebuilder:default=unassigned + HostRef string `json:"hostRef"` + + // +kubebuilder:validation:Optional + IPAddresses map[string]string `json:"ipaddresses"` + + // +kubebuilder:default=false + // Host annotated for deletion + AnnotatedForDeletion bool `json:"annotatedForDeletion"` + + UserDataSecretName string `json:"userDataSecretName"` + NetworkDataSecretName string `json:"networkDataSecretName"` + CtlplaneIP string `json:"ctlplaneIP"` +} + +// +// SyncIPsetStatus - sync relevant information from IPSet to CR status +// +func SyncIPsetStatus( + cond *shared.Condition, + instanceStatus map[string]HostStatus, + ipsetHostStatus ospdirectorv1beta1.HostStatus, +) HostStatus { + var hostStatus HostStatus + if _, ok := instanceStatus[ipsetHostStatus.Hostname]; !ok { + hostStatus = HostStatus(ipsetHostStatus) + } else { + // Note: + // do not sync all information as other controllers are + // the master for e.g. + // - BMH <-> hostname mapping + // - create of networkDataSecretName and userDataSecretName + hostStatus = instanceStatus[ipsetHostStatus.Hostname] + hostStatus.AnnotatedForDeletion = ipsetHostStatus.AnnotatedForDeletion + // TODO: (mschuppert) remove CtlplaneIP where used (osbms) and replace with hostStatus.IPAddresses[] + hostStatus.CtlplaneIP = ipsetHostStatus.CtlplaneIP + hostStatus.IPAddresses = ipsetHostStatus.IPAddresses + hostStatus.ProvisioningState = ipsetHostStatus.ProvisioningState + + if ipsetHostStatus.HostRef != shared.HostRefInitState { + hostStatus.HostRef = ipsetHostStatus.HostRef + } + } + + hostStatus.ProvisioningState = shared.ProvisioningState(cond.Type) + + return hostStatus +} diff --git a/api/v1beta2/common_webhook.go b/api/v1beta2/common_webhook.go new file mode 100644 index 00000000..0f4e37e7 --- /dev/null +++ b/api/v1beta2/common_webhook.go @@ -0,0 +1,244 @@ +package v1beta2 + +// Because webhooks *MUST* exist within the api/ package, we need to place common +// functions here that might be used across different Kinds' webhooks + +import ( + "context" + "fmt" + "regexp" + + ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + goClient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Client needed for API calls (manager's client, set by first SetupWebhookWithManager() call +// to any particular webhook) +var webhookClient goClient.Client + +// checkRoleNameExists - This function is needed by webhooks for both OpenStackBaremetalSets and VMSets +// to help ensure that role names are unique across a given namespace +func checkRoleNameExists(typeMeta metav1.TypeMeta, objectMeta metav1.ObjectMeta, role string) error { + existing, err := getRoleNames(objectMeta.Namespace) + + if err != nil { + return err + } + + for resourceName, roleName := range existing { + // If the role name is already in use by another resource, this resource is invalid + if resourceName != fmt.Sprintf("%s/%s", typeMeta.Kind, objectMeta.Name) && roleName == role { + return fmt.Errorf("Role \"%s\" is already in use by %s", roleName, resourceName) + } + } + + return nil +} + +func getRoleNames(namespace string) (map[string]string, error) { + found := map[string]string{} + + // Get OpenStackBaremetalSet role names + baremetalSetsList := &ospdirectorv1beta1.OpenStackBaremetalSetList{} + listOpts := []goClient.ListOption{ + goClient.InNamespace(namespace), + } + + err := webhookClient.List(context.TODO(), baremetalSetsList, listOpts...) + if err != nil { + return nil, err + } + + for _, bmSet := range baremetalSetsList.Items { + found[fmt.Sprintf("%s/%s", bmSet.Kind, bmSet.Name)] = bmSet.Spec.RoleName + } + + // Get VMSet role names + vmSetsList := &OpenStackVMSetList{} + + err = webhookClient.List(context.TODO(), vmSetsList, listOpts...) + if err != nil { + return nil, err + } + + for _, vmSet := range vmSetsList.Items { + found[fmt.Sprintf("%s/%s", vmSet.Kind, vmSet.Name)] = vmSet.Spec.RoleName + } + + return found, nil +} + +// validateRootDisk - validate configured rootdisk +func validateRootDisk(newDisk OpenStackVMSetDisk, currentDisk OpenStackVMSetDisk) error { + // + // validate DiskSize don't change + // + if err := diskSizeChanged(newDisk.Name, newDisk.DiskSize, currentDisk.DiskSize); err != nil { + return err + } + + // + // validate StorageAccessMode don't change + // + if err := storageAccessModeChanged(newDisk.Name, newDisk.StorageAccessMode, currentDisk.StorageAccessMode); err != nil { + return err + } + + // + // validate StorageClass don't change + // + if err := storageClassChanged(newDisk.Name, newDisk.StorageClass, currentDisk.StorageClass); err != nil { + return err + } + + // + // validate StorageVolumeMode don't change + // + if err := storageVolumeModeChanged(newDisk.Name, newDisk.StorageVolumeMode, currentDisk.StorageVolumeMode); err != nil { + return err + } + + // + // validate BaseImageVolumeName don't change + // + if err := baseImageVolumeNameChanged(newDisk.Name, newDisk.BaseImageVolumeName, currentDisk.BaseImageVolumeName); err != nil { + return err + } + + return nil +} + +// validateAdditionalDisks - validate configured disks +func validateAdditionalDisks(newDisks []OpenStackVMSetDisk, currentDisks []OpenStackVMSetDisk) error { + disks := map[string]OpenStackVMSetDisk{} + curDisks := map[string]OpenStackVMSetDisk{} + + if len(currentDisks) > 0 { + for _, curDisk := range currentDisks { + curDisks[curDisk.Name] = curDisk + } + } + + for _, disk := range newDisks { + // + // validate Disk Name is not 'rootdisk' as this is reserved for the boot disk + // + if disk.Name == "rootdisk" { + return fmt.Errorf(fmt.Sprintf("disk names must not be 'rootdisk' - %v", disk)) + } + + // + // validate Disk Name consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character + // + var validDiskName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + if !validDiskName.MatchString(disk.Name) { + return fmt.Errorf(fmt.Sprintf("disk names must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character - %s", disk.Name)) + } + + // + // validate that disk name is uniq within the VM role + // + if _, ok := disks[disk.Name]; !ok { + disks[disk.Name] = disk + } else { + return fmt.Errorf(fmt.Sprintf("disk names must be uniq within a VM role - %v : %v", disks[disk.Name], disk)) + } + + // + // validations on CR update + // + if curDisk, ok := curDisks[disk.Name]; ok { + // + // validate DiskSize don't change + // + if err := diskSizeChanged(disk.Name, disk.DiskSize, curDisk.DiskSize); err != nil { + return err + } + + // + // validate StorageAccessMode don't change + // + if err := storageAccessModeChanged(disk.Name, disk.StorageAccessMode, curDisk.StorageAccessMode); err != nil { + return err + } + + // + // validate StorageClass don't change + // + if err := storageClassChanged(disk.Name, disk.StorageClass, curDisk.StorageClass); err != nil { + return err + } + + // + // validate StorageVolumeMode don't change + // + if err := storageVolumeModeChanged(disk.Name, disk.StorageVolumeMode, curDisk.StorageVolumeMode); err != nil { + return err + } + } + + } + + return nil +} + +// diskSizeChanged - +func diskSizeChanged(diskName string, diskSize uint32, curDiskSize uint32) error { + // + // validate DiskSize don't change + // + if diskSize != curDiskSize { + return fmt.Errorf(fmt.Sprintf("disk size must not change %s - new %v / current %v", diskName, diskSize, curDiskSize)) + } + + return nil +} + +// storageAccessModeChanged - +func storageAccessModeChanged(diskName string, accessMode string, curAccessMode string) error { + // + // validate StorageAccessMode don't change + // + if accessMode != curAccessMode { + return fmt.Errorf(fmt.Sprintf("StorageAccessMode must not change %s - new %v / current %v", diskName, accessMode, curAccessMode)) + } + + return nil +} + +// storageClassChanged - +func storageClassChanged(diskName string, storageClass string, curStorageClass string) error { + // + // validate StorageClass don't change + // + if storageClass != curStorageClass { + return fmt.Errorf(fmt.Sprintf("StorageClass must not change %s - new %v / current %v", diskName, storageClass, curStorageClass)) + } + + return nil +} + +// storageVolumeModeChanged - +func storageVolumeModeChanged(diskName string, volumeMode string, curVolumeMode string) error { + // + // validate StorageVolumeMode don't change + // + if volumeMode != curVolumeMode { + return fmt.Errorf(fmt.Sprintf("StorageVolumeMode must not change %s - new %v / current %v", diskName, volumeMode, curVolumeMode)) + } + + return nil +} + +// baseImageVolumeNameChanged - +func baseImageVolumeNameChanged(diskName string, volumeName string, curVolumeName string) error { + // + // validate BaseImageVolumeName don't change + // + if volumeName != curVolumeName { + return fmt.Errorf(fmt.Sprintf("BaseImageVolumeName must not change %s - new %v / current %v", diskName, volumeName, curVolumeName)) + } + + return nil +} diff --git a/api/v1beta2/groupversion_info.go b/api/v1beta2/groupversion_info.go new file mode 100644 index 00000000..5780c968 --- /dev/null +++ b/api/v1beta2/groupversion_info.go @@ -0,0 +1,36 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta2 contains API Schema definitions for the osp-director v1beta2 API group +//+kubebuilder:object:generate=true +//+groupName=osp-director.openstack.org +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "osp-director.openstack.org", Version: "v1beta2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1beta2/openstackcontrolplane_types.go b/api/v1beta2/openstackcontrolplane_types.go new file mode 100644 index 00000000..4e03efb5 --- /dev/null +++ b/api/v1beta2/openstackcontrolplane_types.go @@ -0,0 +1,200 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OpenStackControlPlaneSpec defines the desired state of OpenStackControlPlane +type OpenStackControlPlaneSpec struct { + // List of VirtualMachine roles + VirtualMachineRoles map[string]OpenStackVirtualMachineRoleSpec `json:"virtualMachineRoles"` + // +kubebuilder:validation:Optional + // OpenstackClient image. If missing will be set to the configured OPENSTACKCLIENT_IMAGE_URL_DEFAULT in the CSV for the OSP Director Operator. + OpenStackClientImageURL string `json:"openStackClientImageURL,omitempty"` + // OpenStackClientStorageClass storage class + OpenStackClientStorageClass string `json:"openStackClientStorageClass,omitempty"` + // PasswordSecret used to e.g specify root pwd + PasswordSecret string `json:"passwordSecret,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={ctlplane,external} + // OpenStackClientNetworks the name(s) of the OpenStackClientNetworks used to attach the openstackclient to + OpenStackClientNetworks []string `json:"openStackClientNetworks"` + + // +kubebuilder:default=false + // EnableFencing is provided so that users have the option to disable fencing if desired + // FIXME: Defaulting to false until Kubevirt agent merged into RHEL overcloud image + EnableFencing bool `json:"enableFencing"` + + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum={"train","wallaby","16.2","17.0"} + // OpenStackRelease to overwrite OSPrelease auto detection from tripleoclient container image + OpenStackRelease string `json:"openStackRelease"` + + // Idm secret used to register openstack client in IPA + IdmSecret string `json:"idmSecret,omitempty"` + + // Name of the config map containing custom CA certificates to trust + CAConfigMap string `json:"caConfigMap,omitempty"` + + // +kubebuilder:validation:Optional + // AdditionalServiceVIPs - Map of service name <-> subnet nameLower for which a IP should get reserved on + // These are used to create the VirtualFixedIPs environment parameters starting wallaby/OSP17.0. + // Default "Redis": "internal_api", + // "OVNDBs": "internal_api", + // https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/deployment/network_v2.html#service-virtual-ips + // Note: OSP17 networkv2 only + AdditionalServiceVIPs map[string]string `json:"additionalServiceVIPs,omitempty"` +} + +// OpenStackVirtualMachineRoleSpec - defines the desired state of VMs +type OpenStackVirtualMachineRoleSpec struct { + // Number of VMs for the role + RoleCount int `json:"roleCount"` + // number of Cores assigned to the VM + Cores uint32 `json:"cores"` + // amount of Memory in GB used by the VM + Memory uint32 `json:"memory"` + + // +kubebuilder:validation:Optional + // (deprecated) root Disc size in GB - use RootDisk.DiskSize instead + DiskSize uint32 `json:"diskSize,omitempty"` + // +kubebuilder:validation:Optional + // (deprecated) StorageClass to be used for the controller disks - use RootDisk. + StorageClass string `json:"storageClass,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=ReadWriteOnce;ReadWriteMany + // (deprecated) StorageAccessMode - use RootDisk.StorageAccessMode instead + // Virtual machines must have a persistent volume claim (PVC) with a shared ReadWriteMany (RWX) access mode to be live migrated. + StorageAccessMode string `json:"storageAccessMode,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=Block;Filesystem + // (deprecated) StorageVolumeMode - use RootDisk.StorageVolumeMode instead + // When using OpenShift Virtualization with OpenShift Container Platform Container Storage, + // specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks, + // RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs. + // To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block. + StorageVolumeMode string `json:"storageVolumeMode,omitempty"` + // +kubebuilder:validation:Optional + // (deprecated) BaseImageVolumeName used as the base volume for the VM - use RootDisk.BaseImageVolumeName instead + BaseImageVolumeName string `json:"baseImageVolumeName,omitempty"` + + // RootDisk specification of the VM + RootDisk OpenStackVMSetDisk `json:"rootDisk"` + // AdditionalDisks additional disks to add to the VM + AdditionalDisks []OpenStackVMSetDisk `json:"additionalDisks,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=auto;shared + // IOThreadsPolicy - IO thread policy for the domain. Currently valid policies are shared and auto. + // However, if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled and default to shared. + // When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from the vCPUs and placed on the same physical CPU as the QEMU emulator thread. + // An ioThreadsPolicy of shared indicates that KubeVirt should use one thread that will be shared by all disk devices. + IOThreadsPolicy string `json:"ioThreadsPolicy,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Block Multi-Queue is a framework for the Linux block layer that maps Device I/O queries to multiple queues. + // This splits I/O processing up across multiple threads, and therefor multiple CPUs. libvirt recommends that the + // number of queues used should match the number of CPUs allocated for optimal performance. + BlockMultiQueue bool `json:"blockMultiQueue"` + + // +kubebuilder:default=enp2s0 + // Interface to use for ctlplane network + CtlplaneInterface string `json:"ctlplaneInterface"` + + // +kubebuilder:default={ctlplane,external,internalapi,tenant,storage,storagemgmt} + // Networks the name(s) of the OpenStackNetworks used to generate IPs + Networks []string `json:"networks"` + + // RoleName the name of the TripleO role this VM Spec is associated with. If it is a TripleO role, the name must match. + RoleName string `json:"roleName"` + // in case of external functionality, like 3rd party network controllers, set to false to ignore role in rendered overcloud templates. + // +kubebuilder:default=true + IsTripleoRole bool `json:"isTripleoRole,omitempty"` +} + +// OpenStackControlPlaneStatus defines the observed state of OpenStackControlPlane +type OpenStackControlPlaneStatus struct { + VIPStatus map[string]HostStatus `json:"vipStatus,omitempty"` + Conditions shared.ConditionList `json:"conditions,omitempty" optional:"true"` + ProvisioningStatus OpenStackControlPlaneProvisioningStatus `json:"provisioningStatus,omitempty"` + + // OSPVersion the OpenStack version to render templates files + OSPVersion shared.OSPVersion `json:"ospVersion"` +} + +// OpenStackControlPlaneProvisioningStatus represents the overall provisioning state of +// the OpenStackControlPlane (with an optional explanatory message) +type OpenStackControlPlaneProvisioningStatus struct { + State shared.ProvisioningState `json:"state,omitempty"` + Reason string `json:"reason,omitempty"` + DesiredCount int `json:"desiredCount,omitempty"` + ReadyCount int `json:"readyCount,omitempty"` + ClientReady bool `json:"clientReady,omitempty"` +} + +// ControlPlaneProvisioningReason - the reason of the condition for this openstack ctlplane +type ControlPlaneProvisioningReason string + +// IsReady - Is this resource in its fully-configured (quiesced) state? +func (instance *OpenStackControlPlane) IsReady() bool { + return instance.Status.ProvisioningStatus.State == shared.ProvisioningState(shared.ControlPlaneProvisioned) +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:shortName=osctlplane;osctlplanes +// +operator-sdk:csv:customresourcedefinitions:displayName="OpenStack ControlPlane" +// +kubebuilder:printcolumn:name="VMSets Desired",type="integer",JSONPath=".status.provisioningStatus.desiredCount",description="VMSets Desired" +// +kubebuilder:printcolumn:name="VMSets Ready",type="integer",JSONPath=".status.provisioningStatus.readyCount",description="VMSets Ready" +// +kubebuilder:printcolumn:name="Client Ready",type="boolean",JSONPath=".status.provisioningStatus.clientReady",description="Client Ready" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.provisioningStatus.state",description="Status" +// +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=".status.provisioningStatus.reason",description="Reason" + +// OpenStackControlPlane represents a virtualized OpenStack control plane configuration +type OpenStackControlPlane struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OpenStackControlPlaneSpec `json:"spec,omitempty"` + Status OpenStackControlPlaneStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// OpenStackControlPlaneList contains a list of OpenStackControlPlane +type OpenStackControlPlaneList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OpenStackControlPlane `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OpenStackControlPlane{}, &OpenStackControlPlaneList{}) +} + +// GetHostnames - +func (instance OpenStackControlPlane) GetHostnames() map[string]string { + ret := make(map[string]string) + for _, val := range instance.Status.VIPStatus { + ret[val.Hostname] = val.HostRef + } + return ret +} diff --git a/api/v1beta2/openstackcontrolplane_webhook.go b/api/v1beta2/openstackcontrolplane_webhook.go new file mode 100644 index 00000000..eb7a541e --- /dev/null +++ b/api/v1beta2/openstackcontrolplane_webhook.go @@ -0,0 +1,325 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Generated by: +// +// operator-sdk create webhook --group osp-director --version v1beta2 --defaulting --programmatic-validation --conversion --kind OpenStackControlPlane --force +// + +package v1beta2 + +import ( + "context" + "fmt" + "strings" + + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" +) + +var openstackControlPlaneDefaults shared.OpenStackControlPlaneDefaults + +// log is for logging in this package. +var controlplanelog = logf.Log.WithName("controlplane-resource") + +// SetupWebhookWithManager - register this webhook with the controller manager +func (r *OpenStackControlPlane) SetupWebhookWithManager(mgr ctrl.Manager, defaults shared.OpenStackControlPlaneDefaults) error { + + openstackControlPlaneDefaults = defaults + + if webhookClient == nil { + webhookClient = mgr.GetClient() + } + + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta2-openstackcontrolplane,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update,versions=v1beta1;v1beta2,name=mopenstackcontrolplane.kb.io,admissionReviewVersions=v1 + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *OpenStackControlPlane) Default() { + controlplanelog.Info("default", "name", r.Name) + // + // set OpenStackRelease if non provided + // + if r.Spec.OpenStackRelease == "" { + r.Spec.OpenStackRelease = openstackControlPlaneDefaults.OpenStackRelease + r.Status.OSPVersion = shared.OSPVersion(openstackControlPlaneDefaults.OpenStackRelease) + } else { + r.Status.OSPVersion = shared.OSPVersion(r.Spec.OpenStackRelease) + } + controlplanelog.Info(fmt.Sprintf("%s %s using OSP release %s", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.Status.OSPVersion)) + + // + // set default for AdditionalServiceVIPs if non provided in ctlplane spec + // https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/deployment/network_v2.html#service-virtual-ips + // + if r.Status.OSPVersion == shared.TemplateVersion17_0 && r.Spec.AdditionalServiceVIPs == nil { + r.Spec.AdditionalServiceVIPs = map[string]string{ + "Redis": "internal_api", + "OVNDBs": "internal_api", + } + + controlplanelog.Info(fmt.Sprintf("%s %s AdditionalServiceVIPs set %v", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.Spec.AdditionalServiceVIPs)) + } + + // + // set OpenStackNetConfig reference label if not already there + // Note, any rename of the osnetcfg won't be reflected + // + if _, ok := r.GetLabels()[shared.OpenStackNetConfigReconcileLabel]; !ok { + var subnetName string + for _, vmRole := range r.Spec.VirtualMachineRoles { + subnetName = vmRole.Networks[0] + break + } + + labels, err := ospdirectorv1beta1.AddOSNetConfigRefLabel( + webhookClient, + r.Namespace, + subnetName, + r.DeepCopy().GetLabels(), + ) + if err != nil { + controlplanelog.Error(err, fmt.Sprintf("error adding OpenStackNetConfig reference label on %s - %s: %s", r.Kind, r.Name, err)) + } + + r.SetLabels(labels) + controlplanelog.Info(fmt.Sprintf("%s %s labels set to %v", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.GetLabels())) + } + + // + // add labels of all networks used by this CR + // + vipNetList, err := CreateVIPNetworkList(webhookClient, r) + if err != nil { + controlplanelog.Error(err, fmt.Sprintf("error creating VIP network list: %s", err)) + } + + labels := ospdirectorv1beta1.AddOSNetNameLowerLabels( + controlplanelog, + r.DeepCopy().GetLabels(), + vipNetList, + ) + if !equality.Semantic.DeepEqual( + labels, + r.GetLabels(), + ) { + r.SetLabels(labels) + controlplanelog.Info(fmt.Sprintf("%s %s labels set to %v", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.GetLabels())) + } + + // + // set defaults on VM roles + // + for role, vmspec := range r.Spec.VirtualMachineRoles { + + // + // if the VM role spec has the old root disk definition, convert it + // + if _, ok := r.Annotations[shared.RootDiskConvertedAnnotation]; !ok && vmspec.DiskSize > 0 { + convertedRole := OpenStackVirtualMachineRoleSpec{} + + //if role.DiskSize > 0 { + convertedRole.RoleCount = vmspec.RoleCount + convertedRole.Cores = vmspec.Cores + convertedRole.Memory = vmspec.Memory + + // + // Convert root disk + // + convertedRole.RootDisk.DiskSize = vmspec.DiskSize + convertedRole.RootDisk.StorageClass = vmspec.StorageClass + convertedRole.RootDisk.StorageAccessMode = vmspec.StorageAccessMode + convertedRole.RootDisk.StorageVolumeMode = vmspec.StorageVolumeMode + convertedRole.RootDisk.BaseImageVolumeName = vmspec.BaseImageVolumeName + + if len(vmspec.AdditionalDisks) > 0 { + convertedRole.AdditionalDisks = vmspec.AdditionalDisks + } + if strings.EqualFold(vmspec.IOThreadsPolicy, "auto") || strings.EqualFold(vmspec.IOThreadsPolicy, "shared") { + convertedRole.IOThreadsPolicy = vmspec.IOThreadsPolicy + } + convertedRole.BlockMultiQueue = vmspec.BlockMultiQueue + convertedRole.CtlplaneInterface = vmspec.CtlplaneInterface + convertedRole.Networks = vmspec.Networks + convertedRole.RoleName = vmspec.RoleName + convertedRole.IsTripleoRole = vmspec.IsTripleoRole + + controlplanelog.Info(fmt.Sprintf("Converted %s %s role %s: %+v", + r.GetObjectKind().GroupVersionKind().Kind, + r.Name, + role, + convertedRole, + )) + r.Spec.VirtualMachineRoles[role] = convertedRole + + // add RootDiskConvertedAnnotation annotation to skip convertion on next edit + r.SetAnnotations( + shared.MergeStringMaps( + r.GetAnnotations(), + map[string]string{ + shared.RootDiskConvertedAnnotation: "true", + }, + ), + ) + } + } +} + +//+kubebuilder:webhook:path=/validate-osp-director-openstack-org-v1beta2-openstackcontrolplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackcontrolplanes,verbs=create;update;delete,versions=v1beta1;v1beta2,name=vopenstackcontrolplane.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &OpenStackControlPlane{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackControlPlane) ValidateCreate() error { + controlplanelog.Info("validate create", "name", r.Name) + + if err := ospdirectorv1beta1.CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + return err + } + + // + // validate OSP version, right now only 16.2/train and 17.0/wallaby are supported + // + if r.Spec.OpenStackRelease != "" { + var err error + if r.Status.OSPVersion, err = shared.GetOSPVersion(r.Spec.OpenStackRelease); err != nil { + return err + } + } + + controlPlaneList := &OpenStackControlPlaneList{} + + listOpts := []client.ListOption{ + client.InNamespace(r.Namespace), + } + + if err := webhookClient.List(context.TODO(), controlPlaneList, listOpts...); err != nil { + return err + } + + if len(controlPlaneList.Items) >= 1 { + return fmt.Errorf("only one OpenStackControlPlane instance is supported at this time") + } + + // + // Fail early on create if osnetcfg ist not found + // + _, err := ospdirectorv1beta1.GetOsNetCfg(webhookClient, r.GetNamespace(), r.GetLabels()[shared.OpenStackNetConfigReconcileLabel]) + if err != nil { + return fmt.Errorf(fmt.Sprintf("error getting OpenStackNetConfig %s - %s: %s", + r.GetLabels()[shared.OpenStackNetConfigReconcileLabel], + r.Name, + err)) + } + + // + // validate that for all configured subnets an osnet exists + // + for _, vmspec := range r.Spec.VirtualMachineRoles { + // + // validate that for all configured subnets an osnet exists + // + if err := ospdirectorv1beta1.ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { + return err + } + + // + // validate additional disks + // + if err := validateAdditionalDisks(vmspec.AdditionalDisks, []OpenStackVMSetDisk{}); err != nil { + return err + } + + } + + return nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackControlPlane) ValidateUpdate(old runtime.Object) error { + controlplanelog.Info("validate update", "name", r.Name) + + // Get the OpenStackVMSet object + var ok bool + var oldInstance *OpenStackControlPlane + + if oldInstance, ok = old.(*OpenStackControlPlane); !ok { + return fmt.Errorf("runtime object is not an OpenStackControlPlane") + } + + // + // validate OSP version, right now only 16.2/train and 17.0/wallaby are supported + // + if r.Spec.OpenStackRelease != "" { + var err error + if r.Status.OSPVersion, err = shared.GetOSPVersion(r.Spec.OpenStackRelease); err != nil { + return err + } + } + + // + // validate vm roles + // + for role, vmspec := range r.Spec.VirtualMachineRoles { + oldVMSpec := OpenStackVirtualMachineRoleSpec{} + if spec, ok := oldInstance.Spec.VirtualMachineRoles[role]; ok { + oldVMSpec = spec + } + + // + // validate that for all configured subnets an osnet exists + // + if err := ospdirectorv1beta1.ValidateNetworks(r.GetNamespace(), vmspec.Networks); err != nil { + return err + } + + // + // validate rootdisk if its the converted definition format + // + if _, ok := r.Annotations[shared.RootDiskConvertedAnnotation]; ok && oldVMSpec.RootDisk.DiskSize > 0 { + if err := validateRootDisk(vmspec.RootDisk, oldVMSpec.RootDisk); err != nil { + return err + } + } + + // + // validate additional disks + // + if err := validateAdditionalDisks(vmspec.AdditionalDisks, oldVMSpec.AdditionalDisks); err != nil { + return err + } + } + + return nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackControlPlane) ValidateDelete() error { + controlplanelog.Info("validate delete", "name", r.Name) + + return ospdirectorv1beta1.CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) +} diff --git a/api/v1beta2/openstackvmset_types.go b/api/v1beta2/openstackvmset_types.go new file mode 100644 index 00000000..c0dad1d0 --- /dev/null +++ b/api/v1beta2/openstackvmset_types.go @@ -0,0 +1,190 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OpenStackVMSetSpec defines the desired state of an OpenStackVMSet +type OpenStackVMSetSpec struct { + // Number of VMs to configure, 1 or 3 + VMCount int `json:"vmCount"` + // number of Cores assigned to the VMs + Cores uint32 `json:"cores"` + // amount of Memory in GB used by the VMs + Memory uint32 `json:"memory"` + // RootDisk specification of the VM + RootDisk OpenStackVMSetDisk `json:"rootDisk"` + // AdditionalDisks additional disks to add to the VM + AdditionalDisks []OpenStackVMSetDisk `json:"additionalDisks,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=auto;shared + // IOThreadsPolicy - IO thread policy for the domain. Currently valid policies are shared and auto. + // However, if any disk requests a dedicated IOThread, ioThreadsPolicy will be enabled and default to shared. + // When ioThreadsPolicy is set to auto IOThreads will also be "isolated" from the vCPUs and placed on the same physical CPU as the QEMU emulator thread. + // An ioThreadsPolicy of shared indicates that KubeVirt should use one thread that will be shared by all disk devices. + IOThreadsPolicy string `json:"ioThreadsPolicy,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // Block Multi-Queue is a framework for the Linux block layer that maps Device I/O queries to multiple queues. + // This splits I/O processing up across multiple threads, and therefor multiple CPUs. libvirt recommends that the + // number of queues used should match the number of CPUs allocated for optimal performance. + BlockMultiQueue bool `json:"blockMultiQueue"` + + // name of secret holding the stack-admin ssh keys + DeploymentSSHSecret string `json:"deploymentSSHSecret"` + + // +kubebuilder:default=enp2s0 + // Interface to use for ctlplane network + CtlplaneInterface string `json:"ctlplaneInterface"` + + // +kubebuilder:default={ctlplane,external,internalapi,tenant,storage,storagemgmt} + // Networks the name(s) of the OpenStackNetworks used to generate IPs + Networks []string `json:"networks"` + + // RoleName the name of the TripleO role this VM Spec is associated with. If it is a TripleO role, the name must match. + RoleName string `json:"roleName"` + // in case of external functionality, like 3rd party network controllers, set to false to ignore role in rendered overcloud templates. + IsTripleoRole bool `json:"isTripleoRole"` + // PasswordSecret the name of the secret used to optionally set the root pwd by adding + // NodeRootPassword: + // to the secret data + PasswordSecret string `json:"passwordSecret,omitempty"` + + // BootstrapDNS - initial DNS nameserver values to set on the VM when they are provisioned. + // Note that subsequent TripleO deployment will overwrite these values + BootstrapDNS []string `json:"bootstrapDns,omitempty"` + DNSSearchDomains []string `json:"dnsSearchDomains,omitempty"` +} + +// OpenStackVMSetDisk defines additional disk properties +type OpenStackVMSetDisk struct { + // Name of the disk, e.g. used to do the PVC request. + // Must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character + Name string `json:"name"` + // Disc size in GB + DiskSize uint32 `json:"diskSize"` + // StorageClass to be used for the disk + StorageClass string `json:"storageClass,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=ReadWriteMany + // +kubebuilder:validation:Enum=ReadWriteOnce;ReadWriteMany + // StorageAccessMode - Virtual machines must have a persistent volume claim (PVC) + // with a shared ReadWriteMany (RWX) access mode to be live migrated. + StorageAccessMode string `json:"storageAccessMode,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=Filesystem + // +kubebuilder:validation:Enum=Block;Filesystem + // StorageVolumeMode - When using OpenShift Virtualization with OpenShift Container Platform Container Storage, + // specify RBD block mode persistent volume claims (PVCs) when creating virtual machine disks. With virtual machine disks, + // RBD block mode volumes are more efficient and provide better performance than Ceph FS or RBD filesystem-mode PVCs. + // To specify RBD block mode PVCs, use the 'ocs-storagecluster-ceph-rbd' storage class and VolumeMode: Block. + StorageVolumeMode string `json:"storageVolumeMode"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // DedicatedIOThread - Disks with dedicatedIOThread set to true will be allocated an exclusive thread. + // This is generally useful if a specific Disk is expected to have heavy I/O traffic, e.g. a database spindle. + DedicatedIOThread bool `json:"dedicatedIOThread"` + // BaseImageVolumeName used as the base volume for the rootdisk of the VM + BaseImageVolumeName string `json:"baseImageVolumeName"` +} + +// OpenStackVMSetStatus defines the observed state of OpenStackVMSet +type OpenStackVMSetStatus struct { + // BaseImageDVReady is the status of the BaseImage DataVolume + BaseImageDVReady bool `json:"baseImageDVReady,omitempty"` + Conditions shared.ConditionList `json:"conditions,omitempty" optional:"true"` + ProvisioningStatus OpenStackVMSetProvisioningStatus `json:"provisioningStatus,omitempty"` + // VMpods are the names of the kubevirt controller vm pods + VMpods []string `json:"vmpods,omitempty"` + VMHosts map[string]HostStatus `json:"vmHosts,omitempty"` +} + +// OpenStackVMSetProvisioningStatus represents the overall provisioning state of all VMs in +// the OpenStackVMSet (with an optional explanatory message) +type OpenStackVMSetProvisioningStatus struct { + State shared.ProvisioningState `json:"state,omitempty"` + Reason string `json:"reason,omitempty"` + ReadyCount int `json:"readyCount,omitempty"` +} + +// Host - +type Host struct { + Hostname string `json:"hostname"` + HostRef string `json:"hostRef"` + DomainName string `json:"domainName"` + DomainNameUniq string `json:"domainNameUniq"` + IPAddress string `json:"ipAddress"` + NetworkDataSecret string `json:"networkDataSecret"` + BaseImageName string `json:"baseImageName"` + Labels map[string]string `json:"labels"` + NAD map[string]networkv1.NetworkAttachmentDefinition `json:"nad"` +} + +// IsReady - Is this resource in its fully-configured (quiesced) state? +func (instance *OpenStackVMSet) IsReady() bool { + cond := instance.Status.Conditions.InitCondition() + + return cond.Reason == shared.VMSetCondReasonProvisioned || cond.Reason == shared.VMSetCondReasonVirtualMachineCountZero +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:resource:shortName=osvmset;osvmsets;osvms +// +operator-sdk:csv:customresourcedefinitions:displayName="OpenStack VMSet" +// +kubebuilder:printcolumn:name="Cores",type="integer",JSONPath=".spec.cores",description="Cores" +// +kubebuilder:printcolumn:name="RAM",type="integer",JSONPath=".spec.memory",description="RAM in GB" +// +kubebuilder:printcolumn:name="RootDisk",type="integer",JSONPath=".spec.rootDisk.diskSize",description="Root Disk Size" +// +kubebuilder:printcolumn:name="Desired",type="integer",JSONPath=".spec.vmCount",description="Desired" +// +kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.provisioningStatus.readyCount",description="Ready" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.provisioningStatus.state",description="Status" +// +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=".status.provisioningStatus.reason",description="Reason" + +// OpenStackVMSet represents a set of virtual machines hosts for a specific role within the Overcloud deployment +type OpenStackVMSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OpenStackVMSetSpec `json:"spec,omitempty"` + Status OpenStackVMSetStatus `json:"status,omitempty"` +} + +// GetHostnames - +func (instance OpenStackVMSet) GetHostnames() map[string]string { + ret := make(map[string]string) + for _, val := range instance.Status.VMHosts { + ret[val.Hostname] = val.HostRef + } + return ret +} + +// +kubebuilder:object:root=true + +// OpenStackVMSetList contains a list of OpenStackVMSet +type OpenStackVMSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OpenStackVMSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OpenStackVMSet{}, &OpenStackVMSetList{}) +} diff --git a/api/v1beta2/openstackvmset_webhook.go b/api/v1beta2/openstackvmset_webhook.go new file mode 100644 index 00000000..191b0d20 --- /dev/null +++ b/api/v1beta2/openstackvmset_webhook.go @@ -0,0 +1,209 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Generated by: +// +// operator-sdk create webhook --group osp-director --version v1beta2 --defaulting --programmatic-validation --kind OpenStackVMSet --force +// + +package v1beta2 + +import ( + "fmt" + + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" +) + +// log is for logging in this package. +var vmsetlog = logf.Log.WithName("vmset-resource") + +// SetupWebhookWithManager - register this webhook with the controller manager +func (r *OpenStackVMSet) SetupWebhookWithManager(mgr ctrl.Manager) error { + if webhookClient == nil { + webhookClient = mgr.GetClient() + } + + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-osp-director-openstack-org-v1beta2-openstackvmset,mutating=true,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,verbs=create;update,versions=v1beta1;v1beta2,name=mopenstackvmset.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &OpenStackVMSet{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *OpenStackVMSet) Default() { + vmsetlog.Info("default", "name", r.Name) + + // + // set OpenStackNetConfig reference label if not already there + // Note, any rename of the osnetcfg won't be reflected + // + if _, ok := r.GetLabels()[shared.OpenStackNetConfigReconcileLabel]; !ok { + labels, err := ospdirectorv1beta1.AddOSNetConfigRefLabel( + webhookClient, + r.Namespace, + r.Spec.Networks[0], + r.DeepCopy().GetLabels(), + ) + if err != nil { + vmsetlog.Error(err, fmt.Sprintf("error adding OpenStackNetConfig reference label on %s - %s: %s", r.Kind, r.Name, err)) + } + + r.SetLabels(labels) + vmsetlog.Info(fmt.Sprintf("%s %s labels set to %v", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.GetLabels())) + } + + // + // add labels of all networks used by this CR + // + labels := ospdirectorv1beta1.AddOSNetNameLowerLabels( + vmsetlog, + r.DeepCopy().GetLabels(), + r.Spec.Networks, + ) + if !equality.Semantic.DeepEqual( + labels, + r.GetLabels(), + ) { + r.SetLabels(labels) + vmsetlog.Info(fmt.Sprintf("%s %s labels set to %v", r.GetObjectKind().GroupVersionKind().Kind, r.Name, r.GetLabels())) + } + + // + // set spec.domainName , dnsSearchDomains and bootstrapDNS from osnetcfg if not specified + // + osNetCfg, err := ospdirectorv1beta1.GetOsNetCfg(webhookClient, r.GetNamespace(), r.GetLabels()[shared.OpenStackNetConfigReconcileLabel]) + if err != nil { + vmsetlog.Error(err, fmt.Sprintf("error getting OpenStackNetConfig %s - %s: %s", + r.GetLabels()[shared.OpenStackNetConfigReconcileLabel], + r.Name, + err)) + } + + if osNetCfg != nil { + if len(r.Spec.DNSSearchDomains) == 0 && len(osNetCfg.Spec.DNSSearchDomains) > 0 { + r.Spec.DNSSearchDomains = osNetCfg.Spec.DNSSearchDomains + vmsetlog.Info(fmt.Sprintf("Using DNSSearchDomains from %s %s: %v", osNetCfg.GetObjectKind().GroupVersionKind().Kind, osNetCfg.Name, r.Spec.DNSSearchDomains)) + } + if len(r.Spec.BootstrapDNS) == 0 && len(osNetCfg.Spec.DNSServers) > 0 { + r.Spec.BootstrapDNS = osNetCfg.Spec.DNSServers + vmsetlog.Info(fmt.Sprintf("Using BootstrapDNS from %s %s: %v", osNetCfg.GetObjectKind().GroupVersionKind().Kind, osNetCfg.Name, r.Spec.BootstrapDNS)) + } + } + +} + +//+kubebuilder:webhook:path=/validate-osp-director-openstack-org-v1beta2-openstackvmset,mutating=false,failurePolicy=fail,sideEffects=None,groups=osp-director.openstack.org,resources=openstackvmsets,verbs=create;update;delete,versions=v1beta1;v1beta2,name=vopenstackvmset.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &OpenStackVMSet{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackVMSet) ValidateCreate() error { + vmsetlog.Info("validate create", "name", r.Name) + + if err := ospdirectorv1beta1.CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionCreate); err != nil { + return err + } + + // + // Fail early on create if osnetcfg ist not found + // + _, err := ospdirectorv1beta1.GetOsNetCfg(webhookClient, r.GetNamespace(), r.GetLabels()[shared.OpenStackNetConfigReconcileLabel]) + if err != nil { + return fmt.Errorf(fmt.Sprintf("error getting OpenStackNetConfig %s - %s: %s", + r.GetLabels()[shared.OpenStackNetConfigReconcileLabel], + r.Name, + err)) + } + + // + // validate that for all configured subnets an osnet exists + // + if err := ospdirectorv1beta1.ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + return err + } + + // + // validate additional disks + // + if err := validateAdditionalDisks(r.Spec.AdditionalDisks, []OpenStackVMSetDisk{}); err != nil { + return err + } + + return r.validateCr() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackVMSet) ValidateUpdate(old runtime.Object) error { + vmsetlog.Info("validate update", "name", r.Name) + + // Get the OpenStackVMSet object + var ok bool + var oldInstance *OpenStackVMSet + + if oldInstance, ok = old.(*OpenStackVMSet); !ok { + return fmt.Errorf("runtime object is not an OpenStackVMSet") + } + + // + // validate that for all configured subnets an osnet exists + // + if err := ospdirectorv1beta1.ValidateNetworks(r.GetNamespace(), r.Spec.Networks); err != nil { + return err + } + + // + // validate rootdisk + // + if oldInstance.Spec.RootDisk.DiskSize > 0 { + if err := validateRootDisk(r.Spec.RootDisk, oldInstance.Spec.RootDisk); err != nil { + return err + } + } + + // + // validate additional disks + // + if err := validateAdditionalDisks(r.Spec.AdditionalDisks, oldInstance.Spec.AdditionalDisks); err != nil { + return err + } + + return r.validateCr() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *OpenStackVMSet) ValidateDelete() error { + vmsetlog.Info("validate delete", "name", r.Name) + + return ospdirectorv1beta1.CheckBackupOperationBlocksAction(r.Namespace, shared.APIActionDelete) +} + +func (r *OpenStackVMSet) validateCr() error { + if err := checkRoleNameExists(r.TypeMeta, r.ObjectMeta, r.Spec.RoleName); err != nil { + return err + } + + return nil +} diff --git a/api/v1beta2/webhook_suite_test.go b/api/v1beta2/webhook_suite_test.go new file mode 100644 index 00000000..915780a5 --- /dev/null +++ b/api/v1beta2/webhook_suite_test.go @@ -0,0 +1,129 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + //+kubebuilder:scaffold:imports + + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +/* +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc +*/ +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecsWithDefaultAndCustomReporters(t, + "Webhook Suite", + []Reporter{printer.NewlineReporter{}}) +} + +/* +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "config", "webhook")}, + }, + } + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := runtime.NewScheme() + err = AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + LeaderElection: false, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&OpenStackVMSet{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&OpenStackControlPlane{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + conn.Close() + return nil + }).Should(Succeed()) + +}, 60) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) +*/ diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go new file mode 100644 index 00000000..d6fdf80e --- /dev/null +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -0,0 +1,418 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta2 + +import ( + "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Hash) DeepCopyInto(out *Hash) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Hash. +func (in *Hash) DeepCopy() *Hash { + if in == nil { + return nil + } + out := new(Hash) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Host) DeepCopyInto(out *Host) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NAD != nil { + in, out := &in.NAD, &out.NAD + *out = make(map[string]v1.NetworkAttachmentDefinition, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Host. +func (in *Host) DeepCopy() *Host { + if in == nil { + return nil + } + out := new(Host) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostStatus) DeepCopyInto(out *HostStatus) { + *out = *in + if in.IPAddresses != nil { + in, out := &in.IPAddresses, &out.IPAddresses + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostStatus. +func (in *HostStatus) DeepCopy() *HostStatus { + if in == nil { + return nil + } + out := new(HostStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackControlPlane) DeepCopyInto(out *OpenStackControlPlane) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlane. +func (in *OpenStackControlPlane) DeepCopy() *OpenStackControlPlane { + if in == nil { + return nil + } + out := new(OpenStackControlPlane) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackControlPlane) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackControlPlaneList) DeepCopyInto(out *OpenStackControlPlaneList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OpenStackControlPlane, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneList. +func (in *OpenStackControlPlaneList) DeepCopy() *OpenStackControlPlaneList { + if in == nil { + return nil + } + out := new(OpenStackControlPlaneList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackControlPlaneList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackControlPlaneProvisioningStatus) DeepCopyInto(out *OpenStackControlPlaneProvisioningStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneProvisioningStatus. +func (in *OpenStackControlPlaneProvisioningStatus) DeepCopy() *OpenStackControlPlaneProvisioningStatus { + if in == nil { + return nil + } + out := new(OpenStackControlPlaneProvisioningStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackControlPlaneSpec) DeepCopyInto(out *OpenStackControlPlaneSpec) { + *out = *in + if in.VirtualMachineRoles != nil { + in, out := &in.VirtualMachineRoles, &out.VirtualMachineRoles + *out = make(map[string]OpenStackVirtualMachineRoleSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.OpenStackClientNetworks != nil { + in, out := &in.OpenStackClientNetworks, &out.OpenStackClientNetworks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AdditionalServiceVIPs != nil { + in, out := &in.AdditionalServiceVIPs, &out.AdditionalServiceVIPs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneSpec. +func (in *OpenStackControlPlaneSpec) DeepCopy() *OpenStackControlPlaneSpec { + if in == nil { + return nil + } + out := new(OpenStackControlPlaneSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackControlPlaneStatus) DeepCopyInto(out *OpenStackControlPlaneStatus) { + *out = *in + if in.VIPStatus != nil { + in, out := &in.VIPStatus, &out.VIPStatus + *out = make(map[string]HostStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(shared.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ProvisioningStatus = in.ProvisioningStatus +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackControlPlaneStatus. +func (in *OpenStackControlPlaneStatus) DeepCopy() *OpenStackControlPlaneStatus { + if in == nil { + return nil + } + out := new(OpenStackControlPlaneStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSet) DeepCopyInto(out *OpenStackVMSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSet. +func (in *OpenStackVMSet) DeepCopy() *OpenStackVMSet { + if in == nil { + return nil + } + out := new(OpenStackVMSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackVMSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSetDisk) DeepCopyInto(out *OpenStackVMSetDisk) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSetDisk. +func (in *OpenStackVMSetDisk) DeepCopy() *OpenStackVMSetDisk { + if in == nil { + return nil + } + out := new(OpenStackVMSetDisk) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSetList) DeepCopyInto(out *OpenStackVMSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OpenStackVMSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSetList. +func (in *OpenStackVMSetList) DeepCopy() *OpenStackVMSetList { + if in == nil { + return nil + } + out := new(OpenStackVMSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackVMSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSetProvisioningStatus) DeepCopyInto(out *OpenStackVMSetProvisioningStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSetProvisioningStatus. +func (in *OpenStackVMSetProvisioningStatus) DeepCopy() *OpenStackVMSetProvisioningStatus { + if in == nil { + return nil + } + out := new(OpenStackVMSetProvisioningStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSetSpec) DeepCopyInto(out *OpenStackVMSetSpec) { + *out = *in + out.RootDisk = in.RootDisk + if in.AdditionalDisks != nil { + in, out := &in.AdditionalDisks, &out.AdditionalDisks + *out = make([]OpenStackVMSetDisk, len(*in)) + copy(*out, *in) + } + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.BootstrapDNS != nil { + in, out := &in.BootstrapDNS, &out.BootstrapDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DNSSearchDomains != nil { + in, out := &in.DNSSearchDomains, &out.DNSSearchDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSetSpec. +func (in *OpenStackVMSetSpec) DeepCopy() *OpenStackVMSetSpec { + if in == nil { + return nil + } + out := new(OpenStackVMSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVMSetStatus) DeepCopyInto(out *OpenStackVMSetStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(shared.ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ProvisioningStatus = in.ProvisioningStatus + if in.VMpods != nil { + in, out := &in.VMpods, &out.VMpods + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.VMHosts != nil { + in, out := &in.VMHosts, &out.VMHosts + *out = make(map[string]HostStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVMSetStatus. +func (in *OpenStackVMSetStatus) DeepCopy() *OpenStackVMSetStatus { + if in == nil { + return nil + } + out := new(OpenStackVMSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackVirtualMachineRoleSpec) DeepCopyInto(out *OpenStackVirtualMachineRoleSpec) { + *out = *in + out.RootDisk = in.RootDisk + if in.AdditionalDisks != nil { + in, out := &in.AdditionalDisks, &out.AdditionalDisks + *out = make([]OpenStackVMSetDisk, len(*in)) + copy(*out, *in) + } + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackVirtualMachineRoleSpec. +func (in *OpenStackVirtualMachineRoleSpec) DeepCopy() *OpenStackVirtualMachineRoleSpec { + if in == nil { + return nil + } + out := new(OpenStackVirtualMachineRoleSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/osp-director.openstack.org_openstackbackups.yaml b/config/crd/bases/osp-director.openstack.org_openstackbackups.yaml index 9c94319d..e0ebb069 100644 --- a/config/crd/bases/osp-director.openstack.org_openstackbackups.yaml +++ b/config/crd/bases/osp-director.openstack.org_openstackbackups.yaml @@ -855,7 +855,6 @@ spec: it is a TripleO role, the name must match. type: string storageAccessMode: - default: ReadWriteMany description: StorageAccessMode - Virtual machines must have a persistent volume claim (PVC) with a shared ReadWriteMany (RWX) access @@ -869,7 +868,6 @@ spec: controller disks type: string storageVolumeMode: - default: Filesystem description: 'StorageVolumeMode - When using OpenShift Virtualization with OpenShift Container Platform Container Storage, specify diff --git a/config/crd/bases/osp-director.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/osp-director.openstack.org_openstackcontrolplanes.yaml index f33fb415..7ab6ffb0 100644 --- a/config/crd/bases/osp-director.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/osp-director.openstack.org_openstackcontrolplanes.yaml @@ -165,7 +165,6 @@ spec: match. type: string storageAccessMode: - default: ReadWriteMany description: StorageAccessMode - Virtual machines must have a persistent volume claim (PVC) with a shared ReadWriteMany (RWX) access mode to be live migrated. @@ -177,7 +176,6 @@ spec: description: StorageClass to be used for the controller disks type: string storageVolumeMode: - default: Filesystem description: 'StorageVolumeMode - When using OpenShift Virtualization with OpenShift Container Platform Container Storage, specify RBD block mode persistent volume claims (PVCs) when creating @@ -304,6 +302,431 @@ spec: type: object type: object served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: VMSets Desired + jsonPath: .status.provisioningStatus.desiredCount + name: VMSets Desired + type: integer + - description: VMSets Ready + jsonPath: .status.provisioningStatus.readyCount + name: VMSets Ready + type: integer + - description: Client Ready + jsonPath: .status.provisioningStatus.clientReady + name: Client Ready + type: boolean + - description: Status + jsonPath: .status.provisioningStatus.state + name: Status + type: string + - description: Reason + jsonPath: .status.provisioningStatus.reason + name: Reason + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: OpenStackControlPlane represents a virtualized OpenStack control + plane configuration + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OpenStackControlPlaneSpec defines the desired state of OpenStackControlPlane + properties: + additionalServiceVIPs: + additionalProperties: + type: string + description: 'AdditionalServiceVIPs - Map of service name <-> subnet + nameLower for which a IP should get reserved on These are used to + create the VirtualFixedIPs environment parameters starting + wallaby/OSP17.0. Default "Redis": "internal_api", "OVNDBs": "internal_api", + https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/deployment/network_v2.html#service-virtual-ips + Note: OSP17 networkv2 only' + type: object + caConfigMap: + description: Name of the config map containing custom CA certificates + to trust + type: string + enableFencing: + default: false + description: 'EnableFencing is provided so that users have the option + to disable fencing if desired FIXME: Defaulting to false until Kubevirt + agent merged into RHEL overcloud image' + type: boolean + idmSecret: + description: Idm secret used to register openstack client in IPA + type: string + openStackClientImageURL: + description: OpenstackClient image. If missing will be set to the + configured OPENSTACKCLIENT_IMAGE_URL_DEFAULT in the CSV for the + OSP Director Operator. + type: string + openStackClientNetworks: + default: + - ctlplane + - external + description: OpenStackClientNetworks the name(s) of the OpenStackClientNetworks + used to attach the openstackclient to + items: + type: string + type: array + openStackClientStorageClass: + description: OpenStackClientStorageClass storage class + type: string + openStackRelease: + description: OpenStackRelease to overwrite OSPrelease auto detection + from tripleoclient container image + enum: + - train + - wallaby + - "16.2" + - "17.0" + type: string + passwordSecret: + description: PasswordSecret used to e.g specify root pwd + type: string + virtualMachineRoles: + additionalProperties: + description: OpenStackVirtualMachineRoleSpec - defines the desired + state of VMs + properties: + additionalDisks: + description: AdditionalDisks additional disks to add to the + VM + items: + description: OpenStackVMSetDisk defines additional disk properties + properties: + baseImageVolumeName: + description: BaseImageVolumeName used as the base volume + for the rootdisk of the VM + type: string + dedicatedIOThread: + default: false + description: DedicatedIOThread - Disks with dedicatedIOThread + set to true will be allocated an exclusive thread. This + is generally useful if a specific Disk is expected to + have heavy I/O traffic, e.g. a database spindle. + type: boolean + diskSize: + description: Disc size in GB + format: int32 + type: integer + name: + description: Name of the disk, e.g. used to do the PVC + request. Must consist of lower case alphanumeric characters + or '-', and must start and end with an alphanumeric + character + type: string + storageAccessMode: + default: ReadWriteMany + description: StorageAccessMode - Virtual machines must + have a persistent volume claim (PVC) with a shared ReadWriteMany + (RWX) access mode to be live migrated. + enum: + - ReadWriteOnce + - ReadWriteMany + type: string + storageClass: + description: StorageClass to be used for the disk + type: string + storageVolumeMode: + default: Filesystem + description: 'StorageVolumeMode - When using OpenShift + Virtualization with OpenShift Container Platform Container + Storage, specify RBD block mode persistent volume claims + (PVCs) when creating virtual machine disks. With virtual + machine disks, RBD block mode volumes are more efficient + and provide better performance than Ceph FS or RBD filesystem-mode + PVCs. To specify RBD block mode PVCs, use the ''ocs-storagecluster-ceph-rbd'' + storage class and VolumeMode: Block.' + enum: + - Block + - Filesystem + type: string + required: + - baseImageVolumeName + - diskSize + - name + type: object + type: array + baseImageVolumeName: + description: (deprecated) BaseImageVolumeName used as the base + volume for the VM - use RootDisk.BaseImageVolumeName instead + type: string + blockMultiQueue: + default: false + description: Block Multi-Queue is a framework for the Linux + block layer that maps Device I/O queries to multiple queues. + This splits I/O processing up across multiple threads, and + therefor multiple CPUs. libvirt recommends that the number + of queues used should match the number of CPUs allocated for + optimal performance. + type: boolean + cores: + description: number of Cores assigned to the VM + format: int32 + type: integer + ctlplaneInterface: + default: enp2s0 + description: Interface to use for ctlplane network + type: string + diskSize: + description: (deprecated) root Disc size in GB - use RootDisk.DiskSize + instead + format: int32 + type: integer + ioThreadsPolicy: + description: IOThreadsPolicy - IO thread policy for the domain. + Currently valid policies are shared and auto. However, if + any disk requests a dedicated IOThread, ioThreadsPolicy will + be enabled and default to shared. When ioThreadsPolicy is + set to auto IOThreads will also be "isolated" from the vCPUs + and placed on the same physical CPU as the QEMU emulator thread. + An ioThreadsPolicy of shared indicates that KubeVirt should + use one thread that will be shared by all disk devices. + enum: + - auto + - shared + type: string + isTripleoRole: + default: true + description: in case of external functionality, like 3rd party + network controllers, set to false to ignore role in rendered + overcloud templates. + type: boolean + memory: + description: amount of Memory in GB used by the VM + format: int32 + type: integer + networks: + default: + - ctlplane + - external + - internalapi + - tenant + - storage + - storagemgmt + description: Networks the name(s) of the OpenStackNetworks used + to generate IPs + items: + type: string + type: array + roleCount: + description: Number of VMs for the role + type: integer + roleName: + description: RoleName the name of the TripleO role this VM Spec + is associated with. If it is a TripleO role, the name must + match. + type: string + rootDisk: + description: RootDisk specification of the VM + properties: + baseImageVolumeName: + description: BaseImageVolumeName used as the base volume + for the rootdisk of the VM + type: string + dedicatedIOThread: + default: false + description: DedicatedIOThread - Disks with dedicatedIOThread + set to true will be allocated an exclusive thread. This + is generally useful if a specific Disk is expected to + have heavy I/O traffic, e.g. a database spindle. + type: boolean + diskSize: + description: Disc size in GB + format: int32 + type: integer + name: + description: Name of the disk, e.g. used to do the PVC request. + Must consist of lower case alphanumeric characters or + '-', and must start and end with an alphanumeric character + type: string + storageAccessMode: + default: ReadWriteMany + description: StorageAccessMode - Virtual machines must have + a persistent volume claim (PVC) with a shared ReadWriteMany + (RWX) access mode to be live migrated. + enum: + - ReadWriteOnce + - ReadWriteMany + type: string + storageClass: + description: StorageClass to be used for the disk + type: string + storageVolumeMode: + default: Filesystem + description: 'StorageVolumeMode - When using OpenShift Virtualization + with OpenShift Container Platform Container Storage, specify + RBD block mode persistent volume claims (PVCs) when creating + virtual machine disks. With virtual machine disks, RBD + block mode volumes are more efficient and provide better + performance than Ceph FS or RBD filesystem-mode PVCs. + To specify RBD block mode PVCs, use the ''ocs-storagecluster-ceph-rbd'' + storage class and VolumeMode: Block.' + enum: + - Block + - Filesystem + type: string + required: + - baseImageVolumeName + - diskSize + - name + type: object + storageAccessMode: + description: (deprecated) StorageAccessMode - use RootDisk.StorageAccessMode + instead Virtual machines must have a persistent volume claim + (PVC) with a shared ReadWriteMany (RWX) access mode to be + live migrated. + enum: + - ReadWriteOnce + - ReadWriteMany + type: string + storageClass: + description: (deprecated) StorageClass to be used for the controller + disks - use RootDisk. + type: string + storageVolumeMode: + description: '(deprecated) StorageVolumeMode - use RootDisk.StorageVolumeMode + instead When using OpenShift Virtualization with OpenShift + Container Platform Container Storage, specify RBD block mode + persistent volume claims (PVCs) when creating virtual machine + disks. With virtual machine disks, RBD block mode volumes + are more efficient and provide better performance than Ceph + FS or RBD filesystem-mode PVCs. To specify RBD block mode + PVCs, use the ''ocs-storagecluster-ceph-rbd'' storage class + and VolumeMode: Block.' + enum: + - Block + - Filesystem + type: string + required: + - cores + - ctlplaneInterface + - memory + - networks + - roleCount + - roleName + - rootDisk + type: object + description: List of VirtualMachine roles + type: object + required: + - enableFencing + - virtualMachineRoles + type: object + status: + description: OpenStackControlPlaneStatus defines the observed state of + OpenStackControlPlane + properties: + conditions: + description: ConditionList - A list of conditions + items: + description: Condition - A particular overall condition of a certain + resource + properties: + lastHearbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason - Why a particular condition is + true, false or unknown + type: string + status: + type: string + type: + description: ConditionType - A summarizing name for a given + condition + type: string + required: + - status + - type + type: object + type: array + ospVersion: + description: OSPVersion the OpenStack version to render templates + files + type: string + provisioningStatus: + description: OpenStackControlPlaneProvisioningStatus represents the + overall provisioning state of the OpenStackControlPlane (with an + optional explanatory message) + properties: + clientReady: + type: boolean + desiredCount: + type: integer + readyCount: + type: integer + reason: + type: string + state: + description: ProvisioningState - the overall state of all VMs + in this OpenStackVmSet + type: string + type: object + vipStatus: + additionalProperties: + description: HostStatus represents the hostname and IP info for + a specific host + properties: + annotatedForDeletion: + default: false + description: Host annotated for deletion + type: boolean + ctlplaneIP: + type: string + hostRef: + default: unassigned + type: string + hostname: + type: string + ipaddresses: + additionalProperties: + type: string + type: object + networkDataSecretName: + type: string + provisioningState: + description: ProvisioningState - the overall state of all VMs + in this OpenStackVmSet + type: string + userDataSecretName: + type: string + required: + - annotatedForDeletion + - ctlplaneIP + - hostRef + - hostname + - networkDataSecretName + - provisioningState + - userDataSecretName + type: object + type: object + required: + - ospVersion + type: object + type: object + served: true storage: true subresources: status: {} diff --git a/config/crd/bases/osp-director.openstack.org_openstackvmsets.yaml b/config/crd/bases/osp-director.openstack.org_openstackvmsets.yaml index cce65dda..31e160cc 100644 --- a/config/crd/bases/osp-director.openstack.org_openstackvmsets.yaml +++ b/config/crd/bases/osp-director.openstack.org_openstackvmsets.yaml @@ -259,6 +259,352 @@ spec: type: object type: object served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cores + jsonPath: .spec.cores + name: Cores + type: integer + - description: RAM in GB + jsonPath: .spec.memory + name: RAM + type: integer + - description: Root Disk Size + jsonPath: .spec.rootDisk.diskSize + name: RootDisk + type: integer + - description: Desired + jsonPath: .spec.vmCount + name: Desired + type: integer + - description: Ready + jsonPath: .status.provisioningStatus.readyCount + name: Ready + type: integer + - description: Status + jsonPath: .status.provisioningStatus.state + name: Status + type: string + - description: Reason + jsonPath: .status.provisioningStatus.reason + name: Reason + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: OpenStackVMSet represents a set of virtual machines hosts for + a specific role within the Overcloud deployment + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OpenStackVMSetSpec defines the desired state of an OpenStackVMSet + properties: + additionalDisks: + description: AdditionalDisks additional disks to add to the VM + items: + description: OpenStackVMSetDisk defines additional disk properties + properties: + baseImageVolumeName: + description: BaseImageVolumeName used as the base volume for + the rootdisk of the VM + type: string + dedicatedIOThread: + default: false + description: DedicatedIOThread - Disks with dedicatedIOThread + set to true will be allocated an exclusive thread. This is + generally useful if a specific Disk is expected to have heavy + I/O traffic, e.g. a database spindle. + type: boolean + diskSize: + description: Disc size in GB + format: int32 + type: integer + name: + description: Name of the disk, e.g. used to do the PVC request. + Must consist of lower case alphanumeric characters or '-', + and must start and end with an alphanumeric character + type: string + storageAccessMode: + default: ReadWriteMany + description: StorageAccessMode - Virtual machines must have + a persistent volume claim (PVC) with a shared ReadWriteMany + (RWX) access mode to be live migrated. + enum: + - ReadWriteOnce + - ReadWriteMany + type: string + storageClass: + description: StorageClass to be used for the disk + type: string + storageVolumeMode: + default: Filesystem + description: 'StorageVolumeMode - When using OpenShift Virtualization + with OpenShift Container Platform Container Storage, specify + RBD block mode persistent volume claims (PVCs) when creating + virtual machine disks. With virtual machine disks, RBD block + mode volumes are more efficient and provide better performance + than Ceph FS or RBD filesystem-mode PVCs. To specify RBD block + mode PVCs, use the ''ocs-storagecluster-ceph-rbd'' storage + class and VolumeMode: Block.' + enum: + - Block + - Filesystem + type: string + required: + - baseImageVolumeName + - diskSize + - name + type: object + type: array + blockMultiQueue: + default: false + description: Block Multi-Queue is a framework for the Linux block + layer that maps Device I/O queries to multiple queues. This splits + I/O processing up across multiple threads, and therefor multiple + CPUs. libvirt recommends that the number of queues used should match + the number of CPUs allocated for optimal performance. + type: boolean + bootstrapDns: + description: BootstrapDNS - initial DNS nameserver values to set on + the VM when they are provisioned. Note that subsequent TripleO deployment + will overwrite these values + items: + type: string + type: array + cores: + description: number of Cores assigned to the VMs + format: int32 + type: integer + ctlplaneInterface: + default: enp2s0 + description: Interface to use for ctlplane network + type: string + deploymentSSHSecret: + description: name of secret holding the stack-admin ssh keys + type: string + dnsSearchDomains: + items: + type: string + type: array + ioThreadsPolicy: + description: IOThreadsPolicy - IO thread policy for the domain. Currently + valid policies are shared and auto. However, if any disk requests + a dedicated IOThread, ioThreadsPolicy will be enabled and default + to shared. When ioThreadsPolicy is set to auto IOThreads will also + be "isolated" from the vCPUs and placed on the same physical CPU + as the QEMU emulator thread. An ioThreadsPolicy of shared indicates + that KubeVirt should use one thread that will be shared by all disk + devices. + enum: + - auto + - shared + type: string + isTripleoRole: + description: in case of external functionality, like 3rd party network + controllers, set to false to ignore role in rendered overcloud templates. + type: boolean + memory: + description: amount of Memory in GB used by the VMs + format: int32 + type: integer + networks: + default: + - ctlplane + - external + - internalapi + - tenant + - storage + - storagemgmt + description: Networks the name(s) of the OpenStackNetworks used to + generate IPs + items: + type: string + type: array + passwordSecret: + description: 'PasswordSecret the name of the secret used to optionally + set the root pwd by adding NodeRootPassword: to + the secret data' + type: string + roleName: + description: RoleName the name of the TripleO role this VM Spec is + associated with. If it is a TripleO role, the name must match. + type: string + rootDisk: + description: RootDisk specification of the VM + properties: + baseImageVolumeName: + description: BaseImageVolumeName used as the base volume for the + rootdisk of the VM + type: string + dedicatedIOThread: + default: false + description: DedicatedIOThread - Disks with dedicatedIOThread + set to true will be allocated an exclusive thread. This is generally + useful if a specific Disk is expected to have heavy I/O traffic, + e.g. a database spindle. + type: boolean + diskSize: + description: Disc size in GB + format: int32 + type: integer + name: + description: Name of the disk, e.g. used to do the PVC request. + Must consist of lower case alphanumeric characters or '-', and + must start and end with an alphanumeric character + type: string + storageAccessMode: + default: ReadWriteMany + description: StorageAccessMode - Virtual machines must have a + persistent volume claim (PVC) with a shared ReadWriteMany (RWX) + access mode to be live migrated. + enum: + - ReadWriteOnce + - ReadWriteMany + type: string + storageClass: + description: StorageClass to be used for the disk + type: string + storageVolumeMode: + default: Filesystem + description: 'StorageVolumeMode - When using OpenShift Virtualization + with OpenShift Container Platform Container Storage, specify + RBD block mode persistent volume claims (PVCs) when creating + virtual machine disks. With virtual machine disks, RBD block + mode volumes are more efficient and provide better performance + than Ceph FS or RBD filesystem-mode PVCs. To specify RBD block + mode PVCs, use the ''ocs-storagecluster-ceph-rbd'' storage class + and VolumeMode: Block.' + enum: + - Block + - Filesystem + type: string + required: + - baseImageVolumeName + - diskSize + - name + type: object + vmCount: + description: Number of VMs to configure, 1 or 3 + type: integer + required: + - cores + - ctlplaneInterface + - deploymentSSHSecret + - isTripleoRole + - memory + - networks + - roleName + - rootDisk + - vmCount + type: object + status: + description: OpenStackVMSetStatus defines the observed state of OpenStackVMSet + properties: + baseImageDVReady: + description: BaseImageDVReady is the status of the BaseImage DataVolume + type: boolean + conditions: + description: ConditionList - A list of conditions + items: + description: Condition - A particular overall condition of a certain + resource + properties: + lastHearbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason - Why a particular condition is + true, false or unknown + type: string + status: + type: string + type: + description: ConditionType - A summarizing name for a given + condition + type: string + required: + - status + - type + type: object + type: array + provisioningStatus: + description: OpenStackVMSetProvisioningStatus represents the overall + provisioning state of all VMs in the OpenStackVMSet (with an optional + explanatory message) + properties: + readyCount: + type: integer + reason: + type: string + state: + description: ProvisioningState - the overall state of all VMs + in this OpenStackVmSet + type: string + type: object + vmHosts: + additionalProperties: + description: HostStatus represents the hostname and IP info for + a specific host + properties: + annotatedForDeletion: + default: false + description: Host annotated for deletion + type: boolean + ctlplaneIP: + type: string + hostRef: + default: unassigned + type: string + hostname: + type: string + ipaddresses: + additionalProperties: + type: string + type: object + networkDataSecretName: + type: string + provisioningState: + description: ProvisioningState - the overall state of all VMs + in this OpenStackVmSet + type: string + userDataSecretName: + type: string + required: + - annotatedForDeletion + - ctlplaneIP + - hostRef + - hostname + - networkDataSecretName + - provisioningState + - userDataSecretName + type: object + type: object + vmpods: + description: VMpods are the names of the kubevirt controller vm pods + items: + type: string + type: array + type: object + type: object + served: true storage: true subresources: status: {} diff --git a/config/manifests/bases/osp-director-operator.clusterserviceversion.yaml b/config/manifests/bases/osp-director-operator.clusterserviceversion.yaml index dd439b8e..cdd8f4a8 100644 --- a/config/manifests/bases/osp-director-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/osp-director-operator.clusterserviceversion.yaml @@ -63,6 +63,12 @@ spec: kind: OpenStackConfigVersion name: openstackconfigversions.osp-director.openstack.org version: v1beta1 + - description: OpenStackControlPlane represents a virtualized OpenStack control + plane configuration + displayName: OpenStack ControlPlane + kind: OpenStackControlPlane + name: openstackcontrolplanes.osp-director.openstack.org + version: v1beta2 - description: OpenStackControlPlane represents a virtualized OpenStack control plane configuration displayName: OpenStack ControlPlane @@ -117,6 +123,12 @@ spec: kind: OpenStackProvisionServer name: openstackprovisionservers.osp-director.openstack.org version: v1beta1 + - description: OpenStackVMSet represents a set of virtual machines hosts for a + specific role within the Overcloud deployment + displayName: OpenStack VMSet + kind: OpenStackVMSet + name: openstackvmsets.osp-director.openstack.org + version: v1beta2 - description: OpenStackVMSet represents a set of virtual machines hosts for a specific role within the Overcloud deployment displayName: OpenStack VMSet diff --git a/config/rbac/openstackcontrolplane_editor_role.yaml b/config/rbac/openstackcontrolplane_editor_role.yaml new file mode 100644 index 00000000..48df8bd1 --- /dev/null +++ b/config/rbac/openstackcontrolplane_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit openstackcontrolplanes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: openstackcontrolplane-editor-role +rules: +- apiGroups: + - osp-director.openstack.org + resources: + - openstackcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - osp-director.openstack.org + resources: + - openstackcontrolplanes/status + verbs: + - get diff --git a/config/rbac/openstackcontrolplane_viewer_role.yaml b/config/rbac/openstackcontrolplane_viewer_role.yaml new file mode 100644 index 00000000..fb85ed82 --- /dev/null +++ b/config/rbac/openstackcontrolplane_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view openstackcontrolplanes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: openstackcontrolplane-viewer-role +rules: +- apiGroups: + - osp-director.openstack.org + resources: + - openstackcontrolplanes + verbs: + - get + - list + - watch +- apiGroups: + - osp-director.openstack.org + resources: + - openstackcontrolplanes/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1473537a..2de087ae 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -222,6 +222,18 @@ rules: - get - patch - update +- apiGroups: + - migration.k8s.io + resources: + - storageversionmigrations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - nmstate.io resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index d3d14b7f..a0287620 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -16,6 +16,8 @@ resources: - osp-director_v1beta1_openstackmacaddress.yaml - osp-director_v1beta1_openstackbackuprequest.yaml - osp-director_v1beta1_openstackbackup.yaml +- osp-director_v1beta2_openstackvmset.yaml +- osp-director_v1beta2_openstackcontrolplane.yaml # +kubebuilder:scaffold:manifestskustomizesamples # These patches will "correct" some values for samples injected into the CSV, # but leave the samples as-is for functional testing purposes diff --git a/config/samples/osp-director_v1beta2_openstackcontrolplane.yaml b/config/samples/osp-director_v1beta2_openstackcontrolplane.yaml new file mode 100644 index 00000000..49ca80aa --- /dev/null +++ b/config/samples/osp-director_v1beta2_openstackcontrolplane.yaml @@ -0,0 +1,43 @@ +apiVersion: osp-director.openstack.org/v1beta2 +kind: OpenStackControlPlane +metadata: + name: overcloud + namespace: openstack +spec: + domainName: ostest.test.metalkube.org + enableFencing: false + openStackClientImageURL: registry.redhat.io/rhosp-rhel8/openstack-tripleoclient:16.2 + openStackClientNetworks: + - ctlplane + - external + - internal_api + openStackClientStorageClass: host-nfs-storageclass + openStackRelease: "16.2" + passwordSecret: userpassword + virtualMachineRoles: + controller: + cores: 6 + ctlplaneInterface: enp2s0 + rootDisk: + diskSize: 50 + baseImageVolumeName: controller-base-img + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + additionalDisks: + - name: dataDisk1 + diskSize: 1 + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + isTripleoRole: true + memory: 20 + networks: + - ctlplane + - external + - internal_api + - storage + - storage_mgmt + - tenant + roleCount: 0 + roleName: Controller diff --git a/config/samples/osp-director_v1beta2_openstackvmset.yaml b/config/samples/osp-director_v1beta2_openstackvmset.yaml new file mode 100644 index 00000000..a28be1b2 --- /dev/null +++ b/config/samples/osp-director_v1beta2_openstackvmset.yaml @@ -0,0 +1,32 @@ +apiVersion: osp-director.openstack.org/v1beta2 +kind: OpenStackVMSet +metadata: + name: customvmset + namespace: openstack +spec: + vmCount: 1 + cores: 2 + memory: 20 + rootDisk: + diskSize: 50 + baseImageVolumeName: controller-base-img + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + additionalDisks: + - name: dataDisk1 + diskSize: 1 + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + - name: dataDisk2 + diskSize: 1 + storageClass: host-nfs-storageclass + storageAccessMode: ReadWriteMany + storageVolumeMode: Filesystem + deploymentSSHSecret: osp-controlplane-ssh-keys + isTripleoRole: true + ctlplaneInterface: enp2s0 #defaults to enp2s0 + networks: + - ctlplane + roleName: SomeCustomRole diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 1ff3c59e..9cf26134 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,6 +1,6 @@ resources: - manifests.yaml -#- service.yaml +- service.yaml configurations: - kustomizeconfig.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index bf9c5caa..587b1ace 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -65,26 +65,6 @@ webhooks: resources: - openstackconfiggenerators sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /mutate-osp-director-openstack-org-v1beta1-openstackcontrolplane - failurePolicy: Fail - name: mopenstackcontrolplane.kb.io - rules: - - apiGroups: - - osp-director.openstack.org - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - openstackcontrolplanes - sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -231,7 +211,28 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-osp-director-openstack-org-v1beta1-openstackvmset + path: /mutate-osp-director-openstack-org-v1beta2-openstackcontrolplane + failurePolicy: Fail + name: mopenstackcontrolplane.kb.io + rules: + - apiGroups: + - osp-director.openstack.org + apiVersions: + - v1beta1 + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - openstackcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-osp-director-openstack-org-v1beta2-openstackvmset failurePolicy: Fail name: mopenstackvmset.kb.io rules: @@ -239,6 +240,7 @@ webhooks: - osp-director.openstack.org apiVersions: - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -334,27 +336,6 @@ webhooks: resources: - openstackclients sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: webhook-service - namespace: system - path: /validate-osp-director-openstack-org-v1beta1-openstackcontrolplane - failurePolicy: Fail - name: vopenstackcontrolplane.kb.io - rules: - - apiGroups: - - osp-director.openstack.org - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - - DELETE - resources: - - openstackcontrolplanes - sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -483,7 +464,29 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-osp-director-openstack-org-v1beta1-openstackvmset + path: /validate-osp-director-openstack-org-v1beta2-openstackcontrolplane + failurePolicy: Fail + name: vopenstackcontrolplane.kb.io + rules: + - apiGroups: + - osp-director.openstack.org + apiVersions: + - v1beta1 + - v1beta2 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - openstackcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-osp-director-openstack-org-v1beta2-openstackvmset failurePolicy: Fail name: vopenstackvmset.kb.io rules: @@ -491,6 +494,7 @@ webhooks: - osp-director.openstack.org apiVersions: - v1beta1 + - v1beta2 operations: - CREATE - UPDATE diff --git a/controllers/openstackbackuprequest_controller.go b/controllers/openstackbackuprequest_controller.go index edc463ec..73608465 100644 --- a/controllers/openstackbackuprequest_controller.go +++ b/controllers/openstackbackuprequest_controller.go @@ -35,6 +35,7 @@ import ( "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackbackup" ) @@ -143,13 +144,13 @@ func (r *OpenStackBackupRequestReconciler) SetupWithManager(mgr ctrl.Manager) er Owns(&ospdirectorv1beta1.OpenStackBackup{}). Owns(&ospdirectorv1beta1.OpenStackBaremetalSet{}). Owns(&ospdirectorv1beta1.OpenStackClient{}). - Owns(&ospdirectorv1beta1.OpenStackControlPlane{}). + Owns(&ospdirectorv1beta2.OpenStackControlPlane{}). Owns(&ospdirectorv1beta1.OpenStackMACAddress{}). Owns(&ospdirectorv1beta1.OpenStackNet{}). Owns(&ospdirectorv1beta1.OpenStackNetAttachment{}). Owns(&ospdirectorv1beta1.OpenStackNetConfig{}). Owns(&ospdirectorv1beta1.OpenStackProvisionServer{}). - Owns(&ospdirectorv1beta1.OpenStackVMSet{}). + Owns(&ospdirectorv1beta2.OpenStackVMSet{}). Complete(r) } diff --git a/controllers/openstackbaremetalset_controller.go b/controllers/openstackbaremetalset_controller.go index 69592418..6d0ea53b 100644 --- a/controllers/openstackbaremetalset_controller.go +++ b/controllers/openstackbaremetalset_controller.go @@ -890,7 +890,7 @@ func (r *OpenStackBaremetalSetReconciler) baremetalHostProvision( // if err != nil && k8s_errors.IsNotFound(err) { for _, bmhStatus = range instance.Status.BaremetalHosts { - if bmhStatus.HostRef == ospdirectorv1beta1.HostRefInitState { + if bmhStatus.HostRef == shared.HostRefInitState { bmhStatus.HostRef = bmh instance.Status.BaremetalHosts[bmhStatus.Hostname] = bmhStatus diff --git a/controllers/openstackconfiggenerator_controller.go b/controllers/openstackconfiggenerator_controller.go index 74fb0df8..41d5df29 100644 --- a/controllers/openstackconfiggenerator_controller.go +++ b/controllers/openstackconfiggenerator_controller.go @@ -44,6 +44,7 @@ import ( "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" common "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" controlplane "github.com/openstack-k8s-operators/osp-director-operator/pkg/controlplane" openstackconfiggenerator "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackconfiggenerator" @@ -167,7 +168,7 @@ func (r *OpenStackConfigGeneratorReconciler) Reconcile(ctx context.Context, req // // unified OSPVersion from ControlPlane CR // which means also get either 16.2 or 17.0 for upstream versions - controlPlane, ctrlResult, err := ospdirectorv1beta1.GetControlPlane(r.Client, instance) + controlPlane, ctrlResult, err := ospdirectorv1beta2.GetControlPlane(r.Client, instance) if err != nil { cond.Message = err.Error() cond.Reason = shared.ControlPlaneReasonNetNotFound @@ -658,7 +659,7 @@ func (r *OpenStackConfigGeneratorReconciler) verifyNodeResourceStatus( msg := "" // check if all osvmset are in status provisioned - vmsetList := &ospdirectorv1beta1.OpenStackVMSetList{} + vmsetList := &ospdirectorv1beta2.OpenStackVMSetList{} listOpts := []client.ListOption{} err := r.List(ctx, vmsetList, listOpts...) @@ -707,7 +708,7 @@ func (r *OpenStackConfigGeneratorReconciler) createFencingEnvironmentFiles( ctx context.Context, instance *ospdirectorv1beta1.OpenStackConfigGenerator, cond *shared.Condition, - controlPlane *ospdirectorv1beta1.OpenStackControlPlane, + controlPlane *ospdirectorv1beta2.OpenStackControlPlane, tripleoTarballCM *corev1.ConfigMap, cmLabels map[string]string, @@ -806,7 +807,7 @@ func (r *OpenStackConfigGeneratorReconciler) createTripleoDeployCM( envVars *map[string]common.EnvSetter, cmLabels map[string]string, ospVersion shared.OSPVersion, - controlPlane *ospdirectorv1beta1.OpenStackControlPlane, + controlPlane *ospdirectorv1beta2.OpenStackControlPlane, tripleoTarballCM *corev1.ConfigMap, ) (*corev1.ConfigMap, error) { // diff --git a/controllers/openstackcontrolplane_controller.go b/controllers/openstackcontrolplane_controller.go index cb4620a9..96ce182c 100644 --- a/controllers/openstackcontrolplane_controller.go +++ b/controllers/openstackcontrolplane_controller.go @@ -36,6 +36,7 @@ import ( "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" common "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" controlplane "github.com/openstack-k8s-operators/osp-director-operator/pkg/controlplane" openstackclient "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackclient" @@ -43,6 +44,7 @@ import ( vmset "github.com/openstack-k8s-operators/osp-director-operator/pkg/vmset" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + storageversionmigrations "sigs.k8s.io/kube-storage-version-migrator/pkg/apis/migration/v1alpha1" ) // OpenStackControlPlaneReconciler reconciles an OpenStackControlPlane object @@ -82,6 +84,7 @@ func (r *OpenStackControlPlaneReconciler) GetScheme() *runtime.Scheme { // +kubebuilder:rbac:groups=osp-director.openstack.org,resources=openstackclients/status,verbs=get;update;patch // +kubebuilder:rbac:groups=osp-director.openstack.org,resources=openstackmacaddresses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=hco.kubevirt.io,namespace=openstack,resources="*",verbs="*" +// +kubebuilder:rbac:groups=migration.k8s.io,resources=storageversionmigrations,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=create;delete;get;list;patch;update;watch // Reconcile - control plane @@ -89,7 +92,7 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr _ = r.Log.WithValues("controlplane", req.NamespacedName) // Fetch the controlplane instance - instance := &ospdirectorv1beta1.OpenStackControlPlane{} + instance := &ospdirectorv1beta2.OpenStackControlPlane{} err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { @@ -108,7 +111,7 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr cond := &shared.Condition{} if instance.Status.VIPStatus == nil { - instance.Status.VIPStatus = map[string]ospdirectorv1beta1.HostStatus{} + instance.Status.VIPStatus = map[string]ospdirectorv1beta2.HostStatus{} } // If we determine that a backup is overriding this reconcile, requeue after a longer delay @@ -164,6 +167,14 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr envVars := make(map[string]common.EnvSetter) + // + // verify if API storageversionmigration is required + // + ctrlResult, err := r.ensureStorageVersionMigration(ctx, instance, cond) + if (err != nil) || (ctrlResult != ctrl.Result{}) { + return ctrlResult, err + } + // // Set the OSP version, the version is usually set in the ctlplane webhook, // so this is mostly for when running local with no webhooks and no OpenStackRelease is provided @@ -204,7 +215,7 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr // // check if specified PasswordSecret secret exists // - ctrlResult, err := r.verifySecretExist(ctx, instance, cond, instance.Spec.PasswordSecret) + ctrlResult, err = r.verifySecretExist(ctx, instance, cond, instance.Spec.PasswordSecret) if (err != nil) || (ctrlResult != ctrl.Result{}) { return ctrlResult, err } @@ -271,7 +282,6 @@ func (r *OpenStackControlPlaneReconciler) Reconcile(ctx context.Context, req ctr // // Calculate overall status // - //var ctlPlaneState ospdirectorv1beta1.ControlPlaneProvisioningState // 1) OpenStackClient pod status clientPod, err := r.Kclient.CoreV1().Pods(instance.Namespace).Get(ctx, osc.Name, metav1.GetOptions{}) @@ -361,7 +371,7 @@ func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) err controller, ok := labels[common.OwnerControllerNameLabelSelector] if ok || controllers[controller] { // get all CRs from the same namespace - crs := &ospdirectorv1beta1.OpenStackControlPlaneList{} + crs := &ospdirectorv1beta2.OpenStackControlPlaneList{} listOpts := []client.ListOption{ client.InNamespace(obj.GetNamespace()), } @@ -385,18 +395,20 @@ func (r *OpenStackControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) err }) return ctrl.NewControllerManagedBy(mgr). - For(&ospdirectorv1beta1.OpenStackControlPlane{}). + For(&ospdirectorv1beta2.OpenStackControlPlane{}). Owns(&corev1.Secret{}). Owns(&ospdirectorv1beta1.OpenStackIPSet{}). - Owns(&ospdirectorv1beta1.OpenStackVMSet{}). + Owns(&ospdirectorv1beta2.OpenStackVMSet{}). Owns(&ospdirectorv1beta1.OpenStackClient{}). + Owns(&storageversionmigrations.StorageVersionMigration{}). + // watch vmset and openstackclient pods in the same namespace // as we want to reconcile if VMs or openstack client pods change Watches(&source.Kind{Type: &corev1.Pod{}}, podWatcher). Complete(r) } -func (r *OpenStackControlPlaneReconciler) getNormalizedStatus(status *ospdirectorv1beta1.OpenStackControlPlaneStatus) *ospdirectorv1beta1.OpenStackControlPlaneStatus { +func (r *OpenStackControlPlaneReconciler) getNormalizedStatus(status *ospdirectorv1beta2.OpenStackControlPlaneStatus) *ospdirectorv1beta2.OpenStackControlPlaneStatus { // // set LastHeartbeatTime and LastTransitionTime to a default value as those @@ -416,7 +428,7 @@ func (r *OpenStackControlPlaneReconciler) getNormalizedStatus(status *ospdirecto // func (r *OpenStackControlPlaneReconciler) createOrGetTripleoPasswords( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, envVars *map[string]common.EnvSetter, ) error { @@ -479,7 +491,7 @@ func (r *OpenStackControlPlaneReconciler) createOrGetTripleoPasswords( // func (r *OpenStackControlPlaneReconciler) createOrGetDeploymentSecret( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, envVars *map[string]common.EnvSetter, ) (*corev1.Secret, error) { @@ -543,7 +555,7 @@ func (r *OpenStackControlPlaneReconciler) createOrGetDeploymentSecret( func (r *OpenStackControlPlaneReconciler) verifySecretExist( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, secretName string, ) (ctrl.Result, error) { @@ -581,7 +593,7 @@ func (r *OpenStackControlPlaneReconciler) verifySecretExist( func (r *OpenStackControlPlaneReconciler) verifyConfigMapExist( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, configMapName string, ) (ctrl.Result, error) { @@ -621,18 +633,18 @@ func (r *OpenStackControlPlaneReconciler) verifyConfigMapExist( // func (r *OpenStackControlPlaneReconciler) createOrUpdateVMSets( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, deploymentSecret *corev1.Secret, -) ([]*ospdirectorv1beta1.OpenStackVMSet, error) { - vmSets := []*ospdirectorv1beta1.OpenStackVMSet{} +) ([]*ospdirectorv1beta2.OpenStackVMSet, error) { + vmSets := []*ospdirectorv1beta2.OpenStackVMSet{} for _, vmRole := range instance.Spec.VirtualMachineRoles { // // Create or update the vmSet CR object // - vmSet := &ospdirectorv1beta1.OpenStackVMSet{ + vmSet := &ospdirectorv1beta2.OpenStackVMSet{ ObjectMeta: metav1.ObjectMeta{ // use the role name as the VM CR name Name: strings.ToLower(vmRole.RoleName), @@ -644,13 +656,12 @@ func (r *OpenStackControlPlaneReconciler) createOrUpdateVMSets( vmSet.Spec.VMCount = vmRole.RoleCount vmSet.Spec.Cores = vmRole.Cores vmSet.Spec.Memory = vmRole.Memory - vmSet.Spec.DiskSize = vmRole.DiskSize - if vmRole.StorageClass != "" { - vmSet.Spec.StorageClass = vmRole.StorageClass + if len(vmSet.Spec.IOThreadsPolicy) > 0 { + vmSet.Spec.IOThreadsPolicy = vmRole.IOThreadsPolicy } - vmSet.Spec.StorageAccessMode = vmRole.StorageAccessMode - vmSet.Spec.StorageVolumeMode = vmRole.StorageVolumeMode - vmSet.Spec.BaseImageVolumeName = vmRole.DeepCopy().BaseImageVolumeName + vmSet.Spec.BlockMultiQueue = vmRole.BlockMultiQueue + vmSet.Spec.RootDisk = vmRole.RootDisk + vmSet.Spec.AdditionalDisks = vmRole.AdditionalDisks vmSet.Spec.DeploymentSSHSecret = deploymentSecret.Name vmSet.Spec.CtlplaneInterface = vmRole.CtlplaneInterface vmSet.Spec.Networks = vmRole.Networks @@ -703,7 +714,7 @@ func (r *OpenStackControlPlaneReconciler) createOrUpdateVMSets( func (r *OpenStackControlPlaneReconciler) createOrUpdateOpenStackClient( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, deploymentSecret *corev1.Secret, ) (*ospdirectorv1beta1.OpenStackClient, error) { @@ -772,7 +783,7 @@ func (r *OpenStackControlPlaneReconciler) createOrUpdateOpenStackClient( func (r *OpenStackControlPlaneReconciler) ensureVIPs( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackControlPlane, + instance *ospdirectorv1beta2.OpenStackControlPlane, cond *shared.Condition, ) (ctrl.Result, error) { // @@ -780,7 +791,7 @@ func (r *OpenStackControlPlaneReconciler) ensureVIPs( // // create list of networks where Spec.VIP == True - vipNetworksList, err := ospdirectorv1beta1.CreateVIPNetworkList(r.Client, instance) + vipNetworksList, err := ospdirectorv1beta2.CreateVIPNetworkList(r.Client, instance) if err != nil { return ctrl.Result{}, err } @@ -848,7 +859,7 @@ func (r *OpenStackControlPlaneReconciler) ensureVIPs( ) for _, status := range ipsetStatus { - hostStatus := ospdirectorv1beta1.SyncIPsetStatus(cond, instance.Status.VIPStatus, status) + hostStatus := ospdirectorv1beta2.SyncIPsetStatus(cond, instance.Status.VIPStatus, status) instance.Status.VIPStatus[status.Hostname] = hostStatus } @@ -876,7 +887,7 @@ func (r *OpenStackControlPlaneReconciler) ensureVIPs( ) for _, status := range ipsetStatus { - hostStatus := ospdirectorv1beta1.SyncIPsetStatus(cond, instance.Status.VIPStatus, status) + hostStatus := ospdirectorv1beta2.SyncIPsetStatus(cond, instance.Status.VIPStatus, status) instance.Status.VIPStatus[status.Hostname] = hostStatus } @@ -888,3 +899,102 @@ func (r *OpenStackControlPlaneReconciler) ensureVIPs( return ctrl.Result{}, nil } + +// +// verify if API storageversionmigration is required +// +func (r *OpenStackControlPlaneReconciler) ensureStorageVersionMigration( + ctx context.Context, + instance *ospdirectorv1beta2.OpenStackControlPlane, + cond *shared.Condition, +) (ctrl.Result, error) { + // if old vmspec.DiskSize is present in the CR the storageversionmigration needs to be performed + for _, vmspec := range instance.Spec.VirtualMachineRoles { + if vmspec.DiskSize > 0 { + // + // get all storageversionmigrations for ths app label for this CRD + // + smList := &storageversionmigrations.StorageVersionMigrationList{} + labelSelectorMap := map[string]string{ + common.OwnerControllerNameLabelSelector: controlplane.AppLabel, + } + + listOpts := []client.ListOption{ + client.MatchingLabels(labelSelectorMap), + } + + if err := r.List(ctx, smList, listOpts...); err != nil { + err = fmt.Errorf("Error listing services for %s: %v", smList.GroupVersionKind().Kind, err) + return ctrl.Result{}, err + } + + // + // if empty storageversionmigrations list, create the storageversionmigration + // + if len(smList.Items) == 0 { + sm := &storageversionmigrations.StorageVersionMigration{} + // plural, add s + resource := fmt.Sprintf("%ss", strings.ToLower(instance.GroupVersionKind().Kind)) + sm.Name = fmt.Sprintf("%s-storage-version-migration", resource) + sm.SetLabels(common.GetLabels(instance, controlplane.AppLabel, map[string]string{})) + + sm.Spec.Resource.Group = strings.ToLower(instance.GroupVersionKind().Group) + sm.Spec.Resource.Resource = resource + sm.Spec.Resource.Version = "v1beta2" + + err := r.Create(ctx, sm) + if err != nil { + return ctrl.Result{}, err + } + } else { + // + // get storageversionmigrations for this API version + // + for _, sm := range smList.Items { + if sm.Spec.Resource.Version == "v1beta2" { + currentCond := &storageversionmigrations.MigrationCondition{} + for i, c := range sm.Status.Conditions { + if c.Status == corev1.ConditionTrue { + currentCond = &sm.Status.Conditions[i] + break + } + } + + if currentCond != nil { + switch currentCond.Type { + case storageversionmigrations.MigrationSucceeded: + cond.Message = fmt.Sprintf("All runtime objects %s migrated to new API version", + strings.ToLower(instance.GroupVersionKind().Group), + ) + cond.Reason = shared.ConditionReason(currentCond.Reason) + cond.Type = shared.ConditionType(currentCond.Type) + + common.LogForObject(r, cond.Message, instance) + + case storageversionmigrations.MigrationRunning: + cond.Message = fmt.Sprintf("waiting for runtime objects %s to be migrated to new API version", + strings.ToLower(instance.GroupVersionKind().Group), + ) + cond.Reason = shared.ConditionReason(currentCond.Reason) + cond.Type = shared.ConditionType(currentCond.Type) + common.LogForObject(r, cond.Message, instance) + + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil + case storageversionmigrations.MigrationFailed: + cond.Message = fmt.Sprintf("storageversionmigration failed for %s", + strings.ToLower(instance.GroupVersionKind().Group), + ) + cond.Reason = shared.ConditionReason(currentCond.Reason) + cond.Type = shared.ConditionType(currentCond.Type) + err := common.WrapErrorForObject(cond.Message, instance, fmt.Errorf(cond.Message)) + + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, err + } + } + } + } + } + } + } + return ctrl.Result{}, nil +} diff --git a/controllers/openstackdeploy_controller.go b/controllers/openstackdeploy_controller.go index 5b06682e..5f14a516 100644 --- a/controllers/openstackdeploy_controller.go +++ b/controllers/openstackdeploy_controller.go @@ -35,6 +35,7 @@ import ( "github.com/go-logr/logr" "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" common "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackdeploy" ) @@ -131,7 +132,7 @@ func (r *OpenStackDeployReconciler) Reconcile(ctx context.Context, req ctrl.Requ // // unified OSPVersion from ControlPlane CR // which means also get either 16.2 or 17.0 for upstream versions - controlPlane, ctrlResult, err := ospdirectorv1beta1.GetControlPlane(r.Client, instance) + controlPlane, ctrlResult, err := ospdirectorv1beta2.GetControlPlane(r.Client, instance) if err != nil { cond.Message = err.Error() cond.Reason = shared.ControlPlaneReasonNetNotFound diff --git a/controllers/openstackipset_controller.go b/controllers/openstackipset_controller.go index 79561640..28f435ee 100644 --- a/controllers/openstackipset_controller.go +++ b/controllers/openstackipset_controller.go @@ -293,7 +293,7 @@ func (r *OpenStackIPSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque for _, hostStatus := range instance.Status.Hosts { // if there is not yet an assigned BMH host, schedule reconcile - if hostStatus.HostRef == ospdirectorv1beta1.HostRefInitState { + if hostStatus.HostRef == shared.HostRefInitState { return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } } diff --git a/controllers/openstacknetconfig_controller.go b/controllers/openstacknetconfig_controller.go index 5506b44a..03712d3e 100644 --- a/controllers/openstacknetconfig_controller.go +++ b/controllers/openstacknetconfig_controller.go @@ -43,6 +43,7 @@ import ( "github.com/go-logr/logr" "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" common "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" openstackclient "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackclient" macaddress "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackmacaddress" @@ -851,7 +852,7 @@ func (r *OpenStackNetConfigReconciler) createOrUpdateOpenStackMACAddress( client.InNamespace(instance.Namespace), client.MatchingLabels(labelSelector), } - vmSetList := &ospdirectorv1beta1.OpenStackVMSetList{} + vmSetList := &ospdirectorv1beta2.OpenStackVMSetList{} if err := r.GetClient().List( ctx, vmSetList, diff --git a/controllers/openstackvmset_controller.go b/controllers/openstackvmset_controller.go index 49c0742b..87ec4cda 100644 --- a/controllers/openstackvmset_controller.go +++ b/controllers/openstackvmset_controller.go @@ -40,6 +40,7 @@ import ( networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" openstackipset "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackipset" openstacknet "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstacknet" @@ -47,15 +48,18 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" - cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "kubevirt.io/client-go/kubecli" + // cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + // virtctl "kubevirt.io/kubevirt/pkg/virtctl/vm" ) // OpenStackVMSetReconciler reconciles a VMSet object type OpenStackVMSetReconciler struct { client.Client - Kclient kubernetes.Interface - Log logr.Logger - Scheme *runtime.Scheme + Kclient kubernetes.Interface + Log logr.Logger + Scheme *runtime.Scheme + KubevirtClient kubecli.KubevirtClient } // GetClient - @@ -68,6 +72,11 @@ func (r *OpenStackVMSetReconciler) GetKClient() kubernetes.Interface { return r.Kclient } +// GetVirtClient - +func (r *OpenStackVMSetReconciler) GetVirtClient() kubecli.KubevirtClient { + return r.KubevirtClient +} + // GetLogger - func (r *OpenStackVMSetReconciler) GetLogger() logr.Logger { return r.Log @@ -102,7 +111,7 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque _ = r.Log.WithValues("vmset", req.NamespacedName) // Fetch the controller VM instance - instance := &ospdirectorv1beta1.OpenStackVMSet{} + instance := &ospdirectorv1beta2.OpenStackVMSet{} err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { @@ -115,6 +124,17 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } + // + // verify API version + // + if instance.Spec.RootDisk.DiskSize == 0 { + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, + fmt.Errorf(fmt.Sprintf("waiting for runtime object %s %s to be migrated to new API version", + instance.Kind, + instance.Name, + )) + } + // // initialize condition // @@ -122,7 +142,7 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque // If VmSet status map is nil, create it if instance.Status.VMHosts == nil { - instance.Status.VMHosts = map[string]ospdirectorv1beta1.HostStatus{} + instance.Status.VMHosts = map[string]ospdirectorv1beta2.HostStatus{} } // @@ -384,7 +404,7 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque } // - // check/update instance status for annotated for deletion marged VMs + // check/update instance status for annotated for deletion marked VMs // err = r.checkVMsAnnotatedForDeletion( ctx, @@ -430,7 +450,7 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque } for _, status := range ipsetStatus { - hostStatus := ospdirectorv1beta1.SyncIPsetStatus(cond, instance.Status.VMHosts, status) + hostStatus := ospdirectorv1beta2.SyncIPsetStatus(cond, instance.Status.VMHosts, status) instance.Status.VMHosts[status.Hostname] = hostStatus } @@ -485,7 +505,7 @@ func (r *OpenStackVMSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } -func (r *OpenStackVMSetReconciler) getNormalizedStatus(status *ospdirectorv1beta1.OpenStackVMSetStatus) *ospdirectorv1beta1.OpenStackVMSetStatus { +func (r *OpenStackVMSetReconciler) getNormalizedStatus(status *ospdirectorv1beta2.OpenStackVMSetStatus) *ospdirectorv1beta2.OpenStackVMSetStatus { // // set LastHeartbeatTime and LastTransitionTime to a default value as those @@ -502,7 +522,7 @@ func (r *OpenStackVMSetReconciler) getNormalizedStatus(status *ospdirectorv1beta func (r *OpenStackVMSetReconciler) generateNamespaceFencingData( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, ) error { // Ensure that a namespace-scoped kubevirt fencing agent service account token secret has been @@ -599,7 +619,7 @@ func (r *OpenStackVMSetReconciler) generateNamespaceFencingData( func (r *OpenStackVMSetReconciler) generateVirtualMachineNetworkData( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, osNetCfg *ospdirectorv1beta1.OpenStackNetConfig, envVars *map[string]common.EnvSetter, @@ -679,7 +699,7 @@ func (r *OpenStackVMSetReconciler) generateVirtualMachineNetworkData( func (r *OpenStackVMSetReconciler) virtualMachineListFinalizerCleanup( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, virtualMachineList *virtv1.VirtualMachineList, ) error { @@ -724,7 +744,7 @@ func (r *OpenStackVMSetReconciler) SetupWithManager(mgr ctrl.Manager) error { // TODO: Myabe use filtering functions here since some resource permissions // are now cluster-scoped? return ctrl.NewControllerManagedBy(mgr). - For(&ospdirectorv1beta1.OpenStackVMSet{}). + For(&ospdirectorv1beta2.OpenStackVMSet{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Secret{}). Owns(&corev1.PersistentVolumeClaim{}). @@ -735,7 +755,7 @@ func (r *OpenStackVMSetReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *OpenStackVMSetReconciler) vmCreateInstance( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, envVars map[string]common.EnvSetter, ctl *ospdirectorv1beta1.Host, @@ -743,9 +763,13 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( ) error { evictionStrategy := virtv1.EvictionStrategyLiveMigrate - fsMode := corev1.PersistentVolumeMode(instance.Spec.StorageVolumeMode) trueValue := true terminationGracePeriodSeconds := int64(0) + // This run strategy ensures that the VM boots upon creation and reboots upon + // failure, but also allows us to issue manual power commands to the Kubevirt API. + // The default strategy, "Always", disallows the direct power command API calls + // that are required by the Kubevirt fencing agent + runStrategy := virtv1.RunStrategyRerunOnFailure // get deployment userdata from secret userdataSecret := fmt.Sprintf("%s-cloudinit", instance.Name) @@ -763,175 +787,103 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( return err } - // dvTemplateSpec for the VM - dvTemplateSpec := virtv1.DataVolumeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: ctl.DomainNameUniq, - Namespace: instance.Namespace, - }, - Spec: cdiv1.DataVolumeSpec{ - PVC: &corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.PersistentVolumeAccessMode(instance.Spec.StorageAccessMode), - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(fmt.Sprintf("%dGi", instance.Spec.DiskSize)), - }, - }, - VolumeMode: &fsMode, - }, - Source: &cdiv1.DataVolumeSource{ - PVC: &cdiv1.DataVolumeSourcePVC{ - Name: ctl.BaseImageName, - Namespace: instance.Namespace, - }, - }, - }, - } - // set StorageClasseName when specified in the CR - if instance.Spec.StorageClass != "" { - dvTemplateSpec.Spec.PVC.StorageClassName = &instance.Spec.StorageClass - } - - disks := []virtv1.Disk{ - { - Name: "rootdisk", - DiskDevice: virtv1.DiskDevice{ - Disk: &virtv1.DiskTarget{ - Bus: "virtio", - }, - }, - }, - { - Name: "cloudinitdisk", - DiskDevice: virtv1.DiskDevice{ - Disk: &virtv1.DiskTarget{ - Bus: "virtio", - }, - }, - }, - { - Name: "fencingdisk", - Serial: "fencingdisk", - DiskDevice: virtv1.DiskDevice{ - Disk: &virtv1.DiskTarget{ - Bus: "virtio", - }, - }, - }, - } - - volumes := []virtv1.Volume{ - { - Name: "rootdisk", - VolumeSource: virtv1.VolumeSource{ - DataVolume: &virtv1.DataVolumeSource{ - Name: ctl.DomainNameUniq, - }, - }, - }, - { - Name: "cloudinitdisk", - VolumeSource: virtv1.VolumeSource{ - CloudInitNoCloud: &virtv1.CloudInitNoCloudSource{ - UserDataSecretRef: &corev1.LocalObjectReference{ - Name: secret.Name, - }, - NetworkDataSecretRef: &corev1.LocalObjectReference{ - Name: ctl.NetworkDataSecret, - }, - }, - }, - }, - { - Name: "fencingdisk", - VolumeSource: virtv1.VolumeSource{ - Secret: &virtv1.SecretVolumeSource{ - SecretName: vmset.KubevirtFencingKubeconfigSecret, - }, - }, - }, - } - labels := common.GetLabels(instance, vmset.AppLabel, map[string]string{ common.OSPHostnameLabelSelector: ctl.Hostname, "kubevirt.io/vm": ctl.DomainName, }) - vmTemplate := virtv1.VirtualMachineInstanceTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: virtv1.VirtualMachineInstanceSpec{ - Hostname: ctl.DomainName, - EvictionStrategy: &evictionStrategy, - TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Domain: virtv1.DomainSpec{ - Devices: virtv1.Devices{ - Disks: disks, - Interfaces: []virtv1.Interface{ + var vm *virtv1.VirtualMachine + var vmTemplate *virtv1.VirtualMachineInstanceTemplateSpec + if vm, err = r.KubevirtClient.VirtualMachine(instance.Namespace).Get(ctl.DomainName, &metav1.GetOptions{}); err != nil { + // of not found, prepare the VirtualMachineInstanceTemplateSpec + if k8s_errors.IsNotFound(err) { + vmTemplate = &virtv1.VirtualMachineInstanceTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctl.DomainName, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: virtv1.VirtualMachineInstanceSpec{ + Hostname: ctl.DomainName, + EvictionStrategy: &evictionStrategy, + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Domain: virtv1.DomainSpec{ + Devices: virtv1.Devices{ + Disks: []virtv1.Disk{}, + Interfaces: []virtv1.Interface{ + { + Name: "default", + Model: "virtio", + InterfaceBindingMethod: virtv1.InterfaceBindingMethod{ + Masquerade: &virtv1.InterfaceMasquerade{}, + }, + }, + }, + NetworkInterfaceMultiQueue: &trueValue, + Rng: &virtv1.Rng{}, + }, + Machine: &virtv1.Machine{ + Type: "", + }, + }, + Volumes: []virtv1.Volume{}, + Networks: []virtv1.Network{ { - Name: "default", - Model: "virtio", - InterfaceBindingMethod: virtv1.InterfaceBindingMethod{ - Masquerade: &virtv1.InterfaceMasquerade{}, + Name: "default", + NetworkSource: virtv1.NetworkSource{ + Pod: &virtv1.PodNetwork{}, }, }, }, - NetworkInterfaceMultiQueue: &trueValue, - Rng: &virtv1.Rng{}, }, - Machine: &virtv1.Machine{ - Type: "", + } + + // VM + vm = &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctl.DomainName, + Namespace: instance.Namespace, }, - }, - Volumes: volumes, - Networks: []virtv1.Network{ - { - Name: "default", - NetworkSource: virtv1.NetworkSource{ - Pod: &virtv1.PodNetwork{}, - }, + Spec: virtv1.VirtualMachineSpec{ + Template: vmTemplate, }, - }, - }, - } + } - if len(instance.Spec.BootstrapDNS) != 0 { - vmTemplate.Spec.DNSPolicy = corev1.DNSNone - vmTemplate.Spec.DNSConfig = &corev1.PodDNSConfig{ - Nameservers: instance.Spec.BootstrapDNS, + } else { + // Error reading the object - requeue the request. + err = common.WrapErrorForObject("Get VirtualMachineInstanceTemplateSpec", vm, err) + return err } - if len(instance.Spec.DNSSearchDomains) != 0 { - vmTemplate.Spec.DNSConfig.Searches = instance.Spec.DNSSearchDomains + // VM + vm = &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctl.DomainName, + Namespace: instance.Namespace, + Labels: common.GetLabels(instance, vmset.AppLabel, map[string]string{}), + }, + Spec: virtv1.VirtualMachineSpec{ + Template: vmTemplate, + }, } } - // This run strategy ensures that the VM boots upon creation and reboots upon - // failure, but also allows us to issue manual power commands to the Kubevirt API. - // The default strategy, "Always", disallows the direct power command API calls - // that are required by the Kubevirt fencing agent - runStrategy := virtv1.RunStrategyRerunOnFailure - - // VM - vm := &virtv1.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{ - Name: ctl.DomainName, - Namespace: instance.Namespace, - Labels: common.GetLabels(instance, vmset.AppLabel, map[string]string{}), - }, - Spec: virtv1.VirtualMachineSpec{ - RunStrategy: &runStrategy, - Template: &vmTemplate, - }, - } - op, err := controllerutil.CreateOrPatch(ctx, r.Client, vm, func() error { + vm.Labels = shared.MergeStringMaps( + vm.GetLabels(), + common.GetLabels(instance, vmset.AppLabel, map[string]string{}), + ) + vm.Spec.RunStrategy = &runStrategy - vm.Labels = common.GetLabels(instance, vmset.AppLabel, map[string]string{}) + if len(instance.Spec.BootstrapDNS) != 0 { + vm.Spec.Template.Spec.DNSPolicy = corev1.DNSNone + vm.Spec.Template.Spec.DNSConfig = &corev1.PodDNSConfig{ + Nameservers: instance.Spec.BootstrapDNS, + } + if len(instance.Spec.DNSSearchDomains) != 0 { + vm.Spec.Template.Spec.DNSConfig.Searches = instance.Spec.DNSSearchDomains + } + } - vm.Spec.DataVolumeTemplates = []virtv1.DataVolumeTemplateSpec{dvTemplateSpec} vm.Spec.Template.Spec.Domain.CPU = &virtv1.CPU{ Cores: instance.Spec.Cores, } @@ -941,7 +893,9 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( }, } + // // merge additional networks + // networks := instance.Spec.Networks // sort networks to get an expected ordering for easier ooo nic template creation sort.Strings(networks) @@ -999,6 +953,120 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( ) } + // + // Disks and storage + // + if instance.Spec.BlockMultiQueue { + vm.Spec.Template.Spec.Domain.Devices.BlockMultiQueue = &trueValue + } + if len(instance.Spec.IOThreadsPolicy) > 0 { + ioThreadsPolicy := virtv1.IOThreadsPolicy(instance.Spec.IOThreadsPolicy) + vm.Spec.Template.Spec.Domain.IOThreadsPolicy = &ioThreadsPolicy + } + + // root, cloudinit and fencing disk + vm.Spec.DataVolumeTemplates = vmset.MergeVMDataVolumes( + vm.Spec.DataVolumeTemplates, + vmset.DataVolumeSetterMap{ + ctl.DomainNameUniq: vmset.DataVolume( + ctl.DomainNameUniq, + instance.Namespace, + instance.Spec.RootDisk.StorageAccessMode, + instance.Spec.RootDisk.DiskSize, + instance.Spec.RootDisk.StorageVolumeMode, + instance.Spec.RootDisk.StorageClass, + ctl.BaseImageName, + ), + }, + instance.Namespace, + ) + + vm.Spec.Template.Spec.Domain.Devices.Disks = vmset.MergeVMDisks( + vm.Spec.Template.Spec.Domain.Devices.Disks, + vmset.DiskSetterMap{ + "rootdisk": vmset.Disk( + "rootdisk", + "virtio", + "", + instance.Spec.RootDisk.DedicatedIOThread, + ), + "cloudinitdisk": vmset.Disk( + "cloudinitdisk", + "virtio", + "", + false, + ), + "fencingdisk": vmset.Disk( + "fencingdisk", + "virtio", + "fencingdisk", + false, + ), + }, + ) + + vm.Spec.Template.Spec.Volumes = vmset.MergeVMVolumes( + vm.Spec.Template.Spec.Volumes, + vmset.VolumeSetterMap{ + "rootdisk": vmset.VolumeSourceDataVolume( + "rootdisk", + ctl.DomainNameUniq, + ), + "cloudinitdisk": vmset.VolumeSourceCloudInitNoCloud( + "cloudinitdisk", + secret.Name, + ctl.NetworkDataSecret, + ), + "fencingdisk": vmset.VolumeSourceSecret( + "fencingdisk", + vmset.KubevirtFencingKubeconfigSecret, + ), + }, + ) + + // merge additional disks + for _, disk := range instance.Spec.AdditionalDisks { + name := strings.ToLower(fmt.Sprintf("%s-%s", ctl.DomainNameUniq, disk.Name)) + + vm.Spec.DataVolumeTemplates = vmset.MergeVMDataVolumes( + vm.Spec.DataVolumeTemplates, + vmset.DataVolumeSetterMap{ + name: vmset.DataVolume( + name, + instance.Namespace, + disk.StorageAccessMode, + disk.DiskSize, + disk.StorageVolumeMode, + disk.StorageClass, + "", + ), + }, + instance.Namespace, + ) + + vm.Spec.Template.Spec.Domain.Devices.Disks = vmset.MergeVMDisks( + vm.Spec.Template.Spec.Domain.Devices.Disks, + vmset.DiskSetterMap{ + name: vmset.Disk( + name, + "virtio", + "", + disk.DedicatedIOThread, + ), + }, + ) + + vm.Spec.Template.Spec.Volumes = vmset.MergeVMVolumes( + vm.Spec.Template.Spec.Volumes, + vmset.VolumeSetterMap{ + name: vmset.VolumeSourceDataVolume( + name, + name, + ), + }, + ) + } + err := controllerutil.SetControllerReference(instance, vm, r.Scheme) if err != nil { cond.Message = fmt.Sprintf("Error set controller reference for %s", vm.Name) @@ -1012,6 +1080,12 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( return nil }) if err != nil { + cond.Message = fmt.Sprintf("Error create/update %s VM %s", vm.Kind, vm.Name) + cond.Reason = shared.VMSetCondReasonKubevirtError + cond.Type = shared.CommonCondTypeError + err = common.WrapErrorForObject(cond.Message, instance, err) + common.LogForObject(r, fmt.Sprintf("%s: %+v", cond.Message, vm), vm) + return err } @@ -1041,7 +1115,7 @@ func (r *OpenStackVMSetReconciler) vmCreateInstance( // func (r *OpenStackVMSetReconciler) getPasswordSecret( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, ) (string, ctrl.Result, error) { @@ -1079,7 +1153,7 @@ func (r *OpenStackVMSetReconciler) getPasswordSecret( // func (r *OpenStackVMSetReconciler) createCloudInitSecret( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, envVars map[string]common.EnvSetter, secretLabels map[string]string, @@ -1115,7 +1189,7 @@ func (r *OpenStackVMSetReconciler) createCloudInitSecret( // func (r *OpenStackVMSetReconciler) verifyNetworkAttachments( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, osNetBindings map[string]ospdirectorv1beta1.AttachType, ) (map[string]networkv1.NetworkAttachmentDefinition, ctrl.Result, error) { @@ -1199,7 +1273,7 @@ func (r *OpenStackVMSetReconciler) verifyNetworkAttachments( // func (r *OpenStackVMSetReconciler) checkVMsAnnotatedForDeletion( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, ) error { // check for deletion marked VMs @@ -1262,7 +1336,7 @@ func (r *OpenStackVMSetReconciler) checkVMsAnnotatedForDeletion( func (r *OpenStackVMSetReconciler) getDeletedVMOSPHostnames( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, ) ([]string, error) { @@ -1308,12 +1382,12 @@ func (r *OpenStackVMSetReconciler) getDeletedVMOSPHostnames( // func (r *OpenStackVMSetReconciler) createBaseImage( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, ) (string, ctrl.Result, error) { baseImageName := fmt.Sprintf("osp-vmset-baseimage-%s", instance.UID[0:4]) - if instance.Spec.BaseImageVolumeName != "" { - baseImageName = instance.Spec.BaseImageVolumeName + if instance.Spec.RootDisk.BaseImageVolumeName != "" { + baseImageName = instance.Spec.RootDisk.BaseImageVolumeName } // wait for the base image conversion job to be finished before we create the VMs @@ -1360,7 +1434,7 @@ func (r *OpenStackVMSetReconciler) createBaseImage( func (r *OpenStackVMSetReconciler) doVMDelete( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, virtualMachineList *virtv1.VirtualMachineList, ) ([]string, error) { @@ -1424,7 +1498,7 @@ func (r *OpenStackVMSetReconciler) doVMDelete( func (r *OpenStackVMSetReconciler) virtualMachineDeprovision( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, virtualMachine *virtv1.VirtualMachine, ) (string, error) { @@ -1471,7 +1545,7 @@ func (r *OpenStackVMSetReconciler) virtualMachineDeprovision( // func (r *OpenStackVMSetReconciler) createNetworkData( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, osNetCfg *ospdirectorv1beta1.OpenStackNetConfig, nadMap map[string]networkv1.NetworkAttachmentDefinition, @@ -1487,7 +1561,7 @@ func (r *OpenStackVMSetReconciler) createNetworkData( }) // Func to help increase DRY below in NetworkData loops - generateNetworkData := func(instance *ospdirectorv1beta1.OpenStackVMSet, vm *ospdirectorv1beta1.HostStatus) error { + generateNetworkData := func(instance *ospdirectorv1beta2.OpenStackVMSet, vm *ospdirectorv1beta2.HostStatus) error { // TODO mschuppert: get ctlplane network name using ooo-ctlplane-network label netName := "ctlplane" @@ -1558,7 +1632,7 @@ func (r *OpenStackVMSetReconciler) createNetworkData( // func (r *OpenStackVMSetReconciler) createVMs( ctx context.Context, - instance *ospdirectorv1beta1.OpenStackVMSet, + instance *ospdirectorv1beta2.OpenStackVMSet, cond *shared.Condition, envVars map[string]common.EnvSetter, osNetBindings map[string]ospdirectorv1beta1.AttachType, diff --git a/go.mod b/go.mod index 07b7ee60..00fda4b8 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( k8s.io/client-go v12.0.0+incompatible k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b kubevirt.io/api v0.51.0 + kubevirt.io/client-go v0.51.0 kubevirt.io/containerized-data-importer-api v1.42.0 sigs.k8s.io/controller-runtime v0.11.0 sigs.k8s.io/yaml v1.3.0 @@ -50,6 +51,7 @@ require ( github.com/coreos/ign-converter v0.0.0-20200629171308-e40a44f244c5 // indirect github.com/coreos/ignition v0.35.0 // indirect github.com/coreos/ignition/v2 v2.3.0 // indirect + github.com/coreos/prometheus-operator v0.38.0 // indirect github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.12.0 // indirect @@ -57,19 +59,24 @@ require ( github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.1.0 // indirect + github.com/go-kit/kit v0.9.0 // indirect + github.com/go-logfmt/logfmt v0.5.0 // indirect github.com/go-logr/zapr v1.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.5.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.5 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect + github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -100,7 +107,7 @@ require ( golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect @@ -118,6 +125,7 @@ require ( k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + sigs.k8s.io/kube-storage-version-migrator v0.0.5 sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect ) diff --git a/go.sum b/go.sum index 252258cd..3c5f35c8 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,10 @@ github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmU github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -239,6 +241,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= @@ -308,6 +312,7 @@ github.com/coreos/ignition/v2 v2.3.0 h1:TK+STbzVe6KZp4tQ2IaNSRMiWX4/diNngep1F7tP github.com/coreos/ignition/v2 v2.3.0/go.mod h1:85dmM/CERMZXNrJsXqtNLIxR/dn8G9qlL1CmEjCugp0= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/prometheus-operator v0.38.0 h1:gF2xYIfO09XLFdyEecND46uihQ2KTaDwTozRZpXLtN4= github.com/coreos/prometheus-operator v0.38.0/go.mod h1:xZC7/TgeC0/mBaJk+1H9dbHaiEvLYHgX6Mi1h40UPh8= github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE= github.com/coreos/vcontext v0.0.0-20191017033345-260217907eb5 h1:DjoHHi6+9J7DGYPvBdmszKZLY+ucx2bnA77jf8KIk9M= @@ -389,6 +394,7 @@ github.com/elazarl/goproxy/ext v0.0.0-20190911111923-ecfe977594f1/go.mod h1:gNh8 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.8.1+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.10.0+incompatible h1:l6Soi8WCOOVAeCo4W98iBFC6Og7/X8bpRt51oNLZ2C8= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= @@ -460,17 +466,20 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-log/log v0.0.0-20181211034820-a514cf01a3eb/go.mod h1:4mBwpdRMFLiuXZDCwU2lKQFsoSCo72j3HqBK9d81N2M= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= @@ -496,6 +505,7 @@ github.com/go-openapi/jsonpointer v0.17.2/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= @@ -503,6 +513,7 @@ github.com/go-openapi/jsonreference v0.17.2/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -520,6 +531,7 @@ github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsd github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.17.2/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -534,6 +546,7 @@ github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= @@ -541,6 +554,7 @@ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2K github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= @@ -583,6 +597,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -609,6 +624,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -744,6 +760,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw= @@ -813,6 +831,7 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -848,6 +867,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -867,6 +887,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc h1:M7bj0RX9dc79YIyptmHm0tPmC/WuIbn+HVeTBDK2KZw= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20200626054723-37f83d1996bc/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= @@ -907,6 +928,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= +github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 h1:t5bmB3Y8nCaLA4aFrIpX0zjHEF/HUkJp6f5rm7BsVzM= +github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= github.com/kubernetes-sigs/kube-storage-version-migrator v0.0.0-20191127225502-51849bc15f17/go.mod h1:enH0BVV+4+DAgWdwSlMefG8bBzTfVMTr1lApzdLZ/cc= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= @@ -930,6 +955,7 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maorfr/helm-plugin-utils v0.0.0-20200216074820-36d2fcf6ae86/go.mod h1:p3gwmRSFqbWw6plBpR0sKl3n3vpu8kX70gvCJKMvvCA= github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= @@ -1057,6 +1083,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= @@ -1569,6 +1596,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1649,6 +1677,7 @@ golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1757,6 +1786,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1770,6 +1800,7 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1829,8 +1860,9 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 h1:M69LAlWZCshgp0QSzyDcSsSIejIEeuaCVpmwcKwyLMk= golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2032,6 +2064,7 @@ google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -2101,6 +2134,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -2168,6 +2202,7 @@ k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJE k8s.io/cri-api v0.23.0/go.mod h1:2edENu3/mkyW3c6fVPPPaVGEFbLRacJizBbSp7ZOLOo= k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20191010091904-7fa3014cb28f/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -2183,11 +2218,13 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.23.0/go.mod h1:b1vpoaTWKZjCzvbe1KXFw3vPbISrghJsg7/RI8oZUME= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20181031203759-72693cb1fadd/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20190320154901-5e45bb682580/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= @@ -2218,8 +2255,11 @@ k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +kubevirt.io/api v0.0.0-20220308214750-7df276b474a4/go.mod h1:RPYFWI69OVi7i6YtW5gHN3fjYsjlRfRilKVNcpxEMmM= kubevirt.io/api v0.51.0 h1:G7nQDBkEzAxdSgpAPzE9ldXBIv8m0OtsgKWIUc+tvOw= kubevirt.io/api v0.51.0/go.mod h1:RPYFWI69OVi7i6YtW5gHN3fjYsjlRfRilKVNcpxEMmM= +kubevirt.io/client-go v0.51.0 h1:Bn6CAsDIMRMew6hN4a62emFTauHPawWIa7DtJ9ZadLU= +kubevirt.io/client-go v0.51.0/go.mod h1:C34HnV4vfllPhCKOWrP3aRmlvyuCqNZUgtvNlaO2Pu0= kubevirt.io/containerized-data-importer-api v1.42.0 h1:rmL8/M3iv/QvkFntRv9HwZzkqZkH0PHZy3p8Jz98ujI= kubevirt.io/containerized-data-importer-api v1.42.0/go.mod h1:Ty5GJ+6nKlpcBKjeebb/e6IrF8bNFgOus9hfuMjEt6A= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= @@ -2247,9 +2287,12 @@ sigs.k8s.io/controller-tools v0.2.2-0.20190919191502-76a25b63325a/go.mod h1:8SNG sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/controller-tools v0.4.0/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= +sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/kube-storage-version-migrator v0.0.5 h1:PumtXIUB3BJ3LnTV/j+owQEybKR2e46lPflC0Sgea2o= +sigs.k8s.io/kube-storage-version-migrator v0.0.5/go.mod h1:igyHfaOB680DSAOk5x/8mcKk6t6y05AhSl7q+eV22NU= sigs.k8s.io/kustomize/api v0.6.3/go.mod h1:mTwLqPB2uqh7LOcYoJKKHIfP5ioS6t3NfZ8lvia4NJY= sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= sigs.k8s.io/kustomize/cmd/config v0.8.2/go.mod h1:NfndXLA1moQGfNOOThNeLvHb23nrZtgf9gdM7QPdub0= diff --git a/main.go b/main.go index 42292ae5..b6c87415 100644 --- a/main.go +++ b/main.go @@ -42,7 +42,12 @@ import ( machinev1beta1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" sriovnetworkv1 "github.com/openshift/sriov-network-operator/api/v1" + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" + "kubevirt.io/client-go/kubecli" + storageversionmigrations "sigs.k8s.io/kube-storage-version-migrator/pkg/apis/migration/v1alpha1" + "github.com/openstack-k8s-operators/osp-director-operator/controllers" //cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" //templatev1 "github.com/openshift/api/template/v1" @@ -68,6 +73,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(ospdirectorv1beta1.AddToScheme(scheme)) + utilruntime.Must(ospdirectorv1beta2.AddToScheme(scheme)) //utilruntime.Must(templatev1.AddToScheme(scheme)) utilruntime.Must(virtv1.AddToScheme(scheme)) utilruntime.Must(nmstate.AddToScheme(scheme)) @@ -76,6 +82,7 @@ func init() { utilruntime.Must(metal3v1alpha1.AddToScheme(scheme)) utilruntime.Must(machinev1beta1.AddToScheme(scheme)) utilruntime.Must(sriovnetworkv1.AddToScheme(scheme)) + utilruntime.Must(storageversionmigrations.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -135,6 +142,12 @@ func main() { os.Exit(1) } + kubevirtClient, err := kubecli.GetKubevirtClientFromRESTConfig(cfg) + if err != nil { + setupLog.Error(err, "") + os.Exit(1) + } + if strings.ToLower(os.Getenv("ENABLE_WEBHOOKS")) != "false" { enableWebhooks = true @@ -156,10 +169,11 @@ func main() { os.Exit(1) } if err = (&controllers.OpenStackVMSetReconciler{ - Client: mgr.GetClient(), - Kclient: kclient, - Log: ctrl.Log.WithName("controllers").WithName("OpenStackVMSet"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Kclient: kclient, + Log: ctrl.Log.WithName("controllers").WithName("OpenStackVMSet"), + Scheme: mgr.GetScheme(), + KubevirtClient: kubevirtClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "OpenStackVMSet") os.Exit(1) @@ -303,7 +317,7 @@ func main() { // // DEFAULTS // - openstackControlPlaneDefaults := ospdirectorv1beta1.OpenStackControlPlaneDefaults{ + openstackControlPlaneDefaults := shared.OpenStackControlPlaneDefaults{ OpenStackRelease: os.Getenv("OPENSTACK_RELEASE_DEFAULT"), } @@ -355,10 +369,19 @@ func main() { os.Exit(1) } + if err = (&ospdirectorv1beta2.OpenStackControlPlane{}).SetupWebhookWithManager(mgr, openstackControlPlaneDefaults); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "OpenStackControlPlane") + os.Exit(1) + } + if err = (&ospdirectorv1beta1.OpenStackVMSet{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "OpenStackVMSet") os.Exit(1) } + if err = (&ospdirectorv1beta2.OpenStackVMSet{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "OpenStackVMSet") + os.Exit(1) + } if err = (&ospdirectorv1beta1.OpenStackConfigGenerator{}).SetupWebhookWithManager(mgr, openstackConfigGeneratorDefaults); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "OpenStackConfigGenerator") diff --git a/pkg/common/hostname.go b/pkg/common/hostname.go index bc8e8a2d..2199b2e6 100644 --- a/pkg/common/hostname.go +++ b/pkg/common/hostname.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ) // HostnameStore - @@ -92,11 +92,11 @@ func CreateOrGetHostname( chosenNumber = len(foundNumbers) } host.Hostname = fmt.Sprintf("%s-%d", strings.ToLower(host.Basename), chosenNumber) - host.HostRef = ospdirectorv1beta1.HostRefInitState + host.HostRef = shared.HostRefInitState } else { // in case of vip there is only one hostname, set to basename host.Hostname = strings.ToLower(host.Basename) - host.HostRef = ospdirectorv1beta1.HostRefInitState + host.HostRef = shared.HostRefInitState } return nil diff --git a/pkg/openstackbackup/funcs.go b/pkg/openstackbackup/funcs.go index c73a6e03..7828ce15 100644 --- a/pkg/openstackbackup/funcs.go +++ b/pkg/openstackbackup/funcs.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/types" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" "github.com/openstack-k8s-operators/osp-director-operator/pkg/openstackconfiggenerator" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,7 +55,7 @@ func GetCRLists( // OpenStackControlPlanes - osCtlPlanes := ospdirectorv1beta1.OpenStackControlPlaneList{} + osCtlPlanes := ospdirectorv1beta2.OpenStackControlPlaneList{} if err := r.GetClient().List(ctx, &osCtlPlanes, listOpts...); err != nil { return crLists, err @@ -132,7 +133,7 @@ func GetCRLists( // OpenStackVMSets - osVms := ospdirectorv1beta1.OpenStackVMSetList{} + osVms := ospdirectorv1beta2.OpenStackVMSetList{} if err := r.GetClient().List(ctx, &osVms, listOpts...); err != nil { return crLists, err @@ -546,7 +547,7 @@ func GetAreResourcesRestored(backup *ospdirectorv1beta1.OpenStackBackup, crLists // OpenStackVMSets for _, desired := range backup.Spec.Crs.OpenStackVMSets.Items { - found := &ospdirectorv1beta1.OpenStackVMSet{} + found := &ospdirectorv1beta2.OpenStackVMSet{} for _, actual := range crLists.OpenStackVMSets.Items { if actual.Name == desired.Name { @@ -562,7 +563,7 @@ func GetAreResourcesRestored(backup *ospdirectorv1beta1.OpenStackBackup, crLists // OpenStackControlPlanes for _, desired := range backup.Spec.Crs.OpenStackControlPlanes.Items { - found := &ospdirectorv1beta1.OpenStackControlPlane{} + found := &ospdirectorv1beta2.OpenStackControlPlane{} for _, actual := range crLists.OpenStackControlPlanes.Items { if actual.Name == desired.Name { @@ -612,14 +613,14 @@ func CleanNamespace( if len(crLists.OpenStackControlPlanes.Items) > 0 { foundRemaining = true - if err := r.GetClient().DeleteAllOf(ctx, &ospdirectorv1beta1.OpenStackControlPlane{}, client.InNamespace(namespace)); err != nil { + if err := r.GetClient().DeleteAllOf(ctx, &ospdirectorv1beta2.OpenStackControlPlane{}, client.InNamespace(namespace)); err != nil { return false, err } } if len(crLists.OpenStackVMSets.Items) > 0 { foundRemaining = true - if err := r.GetClient().DeleteAllOf(ctx, &ospdirectorv1beta1.OpenStackVMSet{}, client.InNamespace(namespace)); err != nil { + if err := r.GetClient().DeleteAllOf(ctx, &ospdirectorv1beta2.OpenStackVMSet{}, client.InNamespace(namespace)); err != nil { return false, err } } diff --git a/pkg/openstackconfiggenerator/configmap.go b/pkg/openstackconfiggenerator/configmap.go index 031b10f5..11ec26df 100644 --- a/pkg/openstackconfiggenerator/configmap.go +++ b/pkg/openstackconfiggenerator/configmap.go @@ -27,6 +27,8 @@ import ( "github.com/openstack-k8s-operators/osp-director-operator/api/shared" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + ospdirectorv1beta2 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta2" + common "github.com/openstack-k8s-operators/osp-director-operator/pkg/common" "github.com/openstack-k8s-operators/osp-director-operator/pkg/controlplane" "k8s.io/apimachinery/pkg/types" @@ -165,7 +167,7 @@ func CreateConfigMapParams( // // get OSPVersion from ControlPlane CR // - controlPlane, _, err := ospdirectorv1beta1.GetControlPlane(r.GetClient(), &instance.ObjectMeta) + controlPlane, _, err := ospdirectorv1beta2.GetControlPlane(r.GetClient(), &instance.ObjectMeta) if err != nil { return templateParameters, rolesMap, err } @@ -516,7 +518,7 @@ func isVMRole( namespace string, ) (bool, bool, error) { - vmset := &ospdirectorv1beta1.OpenStackVMSet{} + vmset := &ospdirectorv1beta2.OpenStackVMSet{} err := r.GetClient().Get(ctx, types.NamespacedName{Name: roleName, Namespace: namespace}, vmset) if err != nil && !k8s_errors.IsNotFound(err) { diff --git a/pkg/vmset/virtualmachine.go b/pkg/vmset/virtualmachine.go index 037d4580..1f001947 100644 --- a/pkg/vmset/virtualmachine.go +++ b/pkg/vmset/virtualmachine.go @@ -20,9 +20,200 @@ import ( "fmt" ospdirectorv1beta1 "github.com/openstack-k8s-operators/osp-director-operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" ) +// DiskSetter - disk setter for virtv1.Disk +type DiskSetter func(*virtv1.Disk) + +// DiskSetterMap - +type DiskSetterMap map[string]DiskSetter + +// Disk - create additional Disk, ATM only virtio +func Disk( + diskName string, + bus string, + serial string, + dedicatedIOThread bool, +) DiskSetter { + return func(disk *virtv1.Disk) { + disk.Name = diskName + if serial != "" { + disk.Serial = serial + } + disk.DiskDevice.Disk = &virtv1.DiskTarget{} + disk.DiskDevice.Disk.Bus = bus + disk.DedicatedIOThread = &dedicatedIOThread + } +} + +// MergeVMDisks - merge new Disk into existing []virtv1.Disk +func MergeVMDisks(disks []virtv1.Disk, newDisks DiskSetterMap) []virtv1.Disk { + for name, f := range newDisks { + updated := false + for i := 0; i < len(disks); i++ { + if disks[i].Name == name { + f(&disks[i]) + updated = true + break + } + } + + if !updated { + disks = append(disks, virtv1.Disk{Name: name}) + f(&disks[len(disks)-1]) + } + } + + return disks +} + +// VolumeSetter - volume setter for virtv1.Volume +type VolumeSetter func(*virtv1.Volume) + +// VolumeSetterMap - +type VolumeSetterMap map[string]VolumeSetter + +// VolumeSourceDataVolume - create additional VolumeSourceDataVolume +func VolumeSourceDataVolume( + volumeName string, + dataVolumeName string, +) VolumeSetter { + return func(volume *virtv1.Volume) { + volume.Name = volumeName + volume.VolumeSource.DataVolume = &virtv1.DataVolumeSource{} + volume.VolumeSource.DataVolume.Name = dataVolumeName + } +} + +// VolumeSourceCloudInitNoCloud - create additional VolumeSourceCloudInitNoCloud +func VolumeSourceCloudInitNoCloud( + volumeName string, + userDataSecretRefName string, + networkDataSecretRefName string, +) VolumeSetter { + return func(volume *virtv1.Volume) { + volume.Name = volumeName + volume.VolumeSource.CloudInitNoCloud = &virtv1.CloudInitNoCloudSource{} + volume.VolumeSource.CloudInitNoCloud.UserDataSecretRef = &corev1.LocalObjectReference{ + Name: userDataSecretRefName, + } + volume.VolumeSource.CloudInitNoCloud.NetworkDataSecretRef = &corev1.LocalObjectReference{ + Name: networkDataSecretRefName, + } + } +} + +// VolumeSourceSecret - create additional VolumeSourceSecret +func VolumeSourceSecret( + volumeName string, + secretName string, +) VolumeSetter { + return func(volume *virtv1.Volume) { + volume.Name = volumeName + volume.VolumeSource.Secret = &virtv1.SecretVolumeSource{} + volume.VolumeSource.Secret.SecretName = secretName + } +} + +// MergeVMVolumes - merge new Volume into existing []virtv1.Volume +func MergeVMVolumes(volumes []virtv1.Volume, newVolumes VolumeSetterMap) []virtv1.Volume { + for name, f := range newVolumes { + updated := false + for i := 0; i < len(volumes); i++ { + if volumes[i].Name == name { + f(&volumes[i]) + updated = true + break + } + } + + if !updated { + volumes = append(volumes, virtv1.Volume{Name: name}) + f(&volumes[len(volumes)-1]) + } + } + + return volumes +} + +// DataVolumeSetter - volume setter for virtv1.DataVolumeTemplateSpec +type DataVolumeSetter func(*virtv1.DataVolumeTemplateSpec) + +// DataVolumeSetterMap - +type DataVolumeSetterMap map[string]DataVolumeSetter + +// DataVolume - create additional DataVolume +func DataVolume( + dataVolumeName string, + namespace string, + pvAccessMode string, + diskSize uint32, + volumeMode string, + storageClass string, + baseImageName string, +) DataVolumeSetter { + volMode := corev1.PersistentVolumeMode(volumeMode) + + return func(dataVolume *virtv1.DataVolumeTemplateSpec) { + dataVolume.ObjectMeta.Name = dataVolumeName + dataVolume.ObjectMeta.Namespace = namespace + + dataVolume.Spec.PVC = &corev1.PersistentVolumeClaimSpec{} + dataVolume.Spec.PVC.AccessModes = []corev1.PersistentVolumeAccessMode{ + corev1.PersistentVolumeAccessMode(pvAccessMode), + } + + dataVolume.Spec.PVC.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(fmt.Sprintf("%dGi", diskSize)), + }, + } + dataVolume.Spec.PVC.VolumeMode = &volMode + if baseImageName != "" { + dataVolume.Spec.Source = &cdiv1.DataVolumeSource{ + PVC: &cdiv1.DataVolumeSourcePVC{ + Name: baseImageName, + Namespace: namespace, + }, + } + } else { + dataVolume.Spec.Source = &cdiv1.DataVolumeSource{ + Blank: &cdiv1.DataVolumeBlankImage{}, + } + } + } +} + +// MergeVMDataVolumes - merge new DataVolume into existing []virtv1.DataVolumeTemplateSpec +func MergeVMDataVolumes(dataVolumes []virtv1.DataVolumeTemplateSpec, newDataVolumes DataVolumeSetterMap, namespace string) []virtv1.DataVolumeTemplateSpec { + for name, f := range newDataVolumes { + updated := false + for i := 0; i < len(dataVolumes); i++ { + if dataVolumes[i].GetObjectMeta().GetName() == name { + f(&dataVolumes[i]) + updated = true + break + } + } + + if !updated { + dataVolumes = append(dataVolumes, virtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }}) + f(&dataVolumes[len(dataVolumes)-1]) + } + } + + return dataVolumes +} + // NetSetter - net setter for virtv1.Network type NetSetter func(*virtv1.Network)