diff --git a/pkg/kubernetes/secret/kubernetes.go b/pkg/kubernetes/secret/kubernetes.go index 9dfa6695bd..063ee35671 100644 --- a/pkg/kubernetes/secret/kubernetes.go +++ b/pkg/kubernetes/secret/kubernetes.go @@ -40,6 +40,10 @@ type getFn func(cli *kubernetes.Clientset, namespace, name string, opts metav1.G // to create secret type createFn func(cli *kubernetes.Clientset, namespace string, secret *corev1.Secret) (*corev1.Secret, error) +// updateFn is a typed function that abstracts +// to update secret +type updateFn func(cli *kubernetes.Clientset, namespace string, secret *corev1.Secret) (*corev1.Secret, error) + // listFn is a typed function that abstracts listing of secret instances type listFn func(cli *kubernetes.Clientset, namespace string, opts metav1.ListOptions) (*corev1.SecretList, error) @@ -68,6 +72,7 @@ type Kubeclient struct { create createFn del deleteFn list listFn + update updateFn } // KubeClientBuildOption defines the abstraction @@ -100,7 +105,11 @@ func (k *Kubeclient) withDefaults() { return cli.CoreV1().Secrets(namespace).List(opts) } } - + if k.update == nil { + k.update = func(cli *kubernetes.Clientset, namespace string, secret *corev1.Secret) (*corev1.Secret, error) { + return cli.CoreV1().Secrets(namespace).Update(secret) + } + } if k.del == nil { k.del = func(cli *kubernetes.Clientset, namespace, name string, opts *metav1.DeleteOptions) error { return cli.CoreV1().Secrets(namespace).Delete(name, opts) @@ -202,3 +211,15 @@ func (k *Kubeclient) List(opts metav1.ListOptions) (*corev1.SecretList, error) { } return k.list(cli, k.namespace, opts) } + +// Update updates and returns updated secret instance +func (k *Kubeclient) Update(secret *corev1.Secret) (*corev1.Secret, error) { + if secret == nil { + return nil, errors.New("failed to update secret: nil secret object") + } + cli, err := k.getClientsetOrCached() + if err != nil { + return nil, errors.Wrapf(err, "failed to update secret %s", secret.Name) + } + return k.update(cli, k.namespace, secret) +} diff --git a/pkg/kubernetes/service/v1alpha1/kubernetes.go b/pkg/kubernetes/service/v1alpha1/kubernetes.go index 5d642f2812..bf992f98fb 100644 --- a/pkg/kubernetes/service/v1alpha1/kubernetes.go +++ b/pkg/kubernetes/service/v1alpha1/kubernetes.go @@ -65,6 +65,13 @@ type createFn func( namespace string, ) (*corev1.Service, error) +// updateFn is a typed function that abstracts delete of service instances +type updateFn func( + cli *kubernetes.Clientset, + service *corev1.Service, + namespace string, +) (*corev1.Service, error) + // patchFn is a typed function that abstracts patch of service instances type patchFn func( cli *kubernetes.Clientset, @@ -92,6 +99,7 @@ type Kubeclient struct { del delFn create createFn patch patchFn + update updateFn } // KubeclientBuildOption defines the abstraction to build a kubeclient instance @@ -170,6 +178,18 @@ func defaultCreate( Create(service) } +// defaultUpdate is the default implementation to update +// a service instance in kubernetes cluster +func defaultUpdate( + cli *kubernetes.Clientset, + service *corev1.Service, + namespace string, +) (*corev1.Service, error) { + return cli.CoreV1(). + Services(namespace). + Update(service) +} + // defaultPatch is the default implementation to patch // a service instance in kubernetes cluster func defaultPatch( @@ -207,6 +227,9 @@ func (k *Kubeclient) withDefaults() { if k.patch == nil { k.patch = defaultPatch } + if k.update == nil { + k.update = defaultUpdate + } } // WithClientset sets the kubernetes client against the kubeclient instance @@ -336,6 +359,23 @@ func (k *Kubeclient) Create(service *corev1.Service) (*corev1.Service, error) { return k.create(cli, service, k.namespace) } +// Update updates a service in specified namespace in kubernetes cluster +func (k *Kubeclient) Update(service *corev1.Service) (*corev1.Service, error) { + if service == nil { + return nil, errors.New("failed to update service: nil service object") + } + cli, err := k.getClientOrCached() + if err != nil { + return nil, errors.Wrapf( + err, + "failed to update service {%s} in namespace {%s}", + service.Name, + service.Namespace, + ) + } + return k.update(cli, service, k.namespace) +} + // Patch patches service object for given name func (k *Kubeclient) Patch( name string, diff --git a/pkg/kubernetes/webhook/validate/v1alpha1/kubernetes.go b/pkg/kubernetes/webhook/validate/v1alpha1/kubernetes.go index ec490df96c..9bab76093c 100644 --- a/pkg/kubernetes/webhook/validate/v1alpha1/kubernetes.go +++ b/pkg/kubernetes/webhook/validate/v1alpha1/kubernetes.go @@ -52,6 +52,14 @@ type createFunc func(cli *kubernetes.Clientset, error, ) +// updateFn is a typed function that abstracts +// to update admissionwebhook configuration +type updateFn func(cli *kubernetes.Clientset, + config *admission.ValidatingWebhookConfiguration) ( + *admission.ValidatingWebhookConfiguration, + error, +) + // Kubeclient enables kubernetes API operations // on upgrade result instance type Kubeclient struct { @@ -66,6 +74,7 @@ type Kubeclient struct { create createFunc get getFunc del delFunc + update updateFn } // KubeclientBuildOption defines the abstraction @@ -103,9 +112,12 @@ func (k *Kubeclient) withDefaults() { k.del = func(cs *kubernetes.Clientset, name string, opts *metav1.DeleteOptions) error { return cs.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(name, opts) } - } - + if k.update == nil { + k.update = func(cs *kubernetes.Clientset, config *admission.ValidatingWebhookConfiguration) (*admission.ValidatingWebhookConfiguration, error) { + return cs.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Update(config) + } + } } // WithClientset sets the kubernetes clientset against @@ -197,3 +209,16 @@ func (k *Kubeclient) Delete(name string, options *metav1.DeleteOptions) error { } return k.del(cli, name, options) } + +// Update updates validatingWebhookConfiguration, and returns the updated +// corresponding validatingWebhookConfiguration object, and an error if there is any. +func (k *Kubeclient) Update(config *admission.ValidatingWebhookConfiguration) (*admission.ValidatingWebhookConfiguration, error) { + if config == nil { + return nil, errors.New("failed to update validating configuration: nil configuration") + } + cs, err := k.getClientOrCached() + if err != nil { + return nil, err + } + return k.update(cs, config) +} diff --git a/pkg/webhook/configuration.go b/pkg/webhook/configuration.go index 1507f8bac6..714958cfec 100644 --- a/pkg/webhook/configuration.go +++ b/pkg/webhook/configuration.go @@ -21,11 +21,13 @@ import ( "os" "strings" + apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" menv "github.com/openebs/maya/pkg/env/v1alpha1" deploy "github.com/openebs/maya/pkg/kubernetes/deployment/appsv1/v1alpha1" secret "github.com/openebs/maya/pkg/kubernetes/secret" svc "github.com/openebs/maya/pkg/kubernetes/service/v1alpha1" validate "github.com/openebs/maya/pkg/kubernetes/webhook/validate/v1alpha1" + "github.com/openebs/maya/pkg/util" "github.com/openebs/maya/pkg/version" "github.com/pkg/errors" "k8s.io/api/admissionregistration/v1beta1" @@ -54,6 +56,10 @@ const ( rootCrt = "ca.crt" ) +type transformSvcFunc func(*corev1.Service) +type transformSecretFunc func(*corev1.Secret) +type transformConfigFunc func(*v1beta1.ValidatingWebhookConfiguration) + var ( // TimeoutSeconds specifies the timeout for this webhook. After the timeout passes, // the webhook call will be ignored or the API call will fail based on the @@ -62,6 +68,12 @@ var ( five = int32(5) // Ignore means that an error calling the webhook is ignored. Ignore = v1beta1.Ignore + // transformation function lists to upgrade webhook resources + transformSecret = []transformSecretFunc{} + transformSvc = []transformSvcFunc{} + transformConfig = []transformConfigFunc{ + addCSPCDeleteRule, + } ) // createWebhookService creates our webhook Service resource if it does not @@ -99,9 +111,9 @@ func createWebhookService( Namespace: namespace, Name: serviceName, Labels: map[string]string{ - "app": "admission-webhook", - "openebs.io/component-name": "admission-webhook-svc", - "openebs.io/version": version.GetVersion(), + "app": "admission-webhook", + "openebs.io/component-name": "admission-webhook-svc", + string(apis.OpenEBSVersionKey): version.GetVersion(), }, OwnerReferences: []metav1.OwnerReference{ownerReference}, }, @@ -163,6 +175,7 @@ func createAdmissionService( Operations: []v1beta1.OperationType{ v1beta1.Create, v1beta1.Update, + v1beta1.Delete, }, Rule: v1beta1.Rule{ APIGroups: []string{"*"}, @@ -190,9 +203,9 @@ func createAdmissionService( ObjectMeta: metav1.ObjectMeta{ Name: validatorWebhook, Labels: map[string]string{ - "app": "admission-webhook", - "openebs.io/component-name": "admission-webhook", - "openebs.io/version": version.GetVersion(), + "app": "admission-webhook", + "openebs.io/component-name": "admission-webhook", + string(apis.OpenEBSVersionKey): version.GetVersion(), }, OwnerReferences: []metav1.OwnerReference{ownerReference}, }, @@ -244,9 +257,9 @@ func createCertsSecret( Name: secretName, Namespace: namespace, Labels: map[string]string{ - "app": "admission-webhook", - "openebs.io/component-name": "admission-webhook", - "openebs.io/version": version.GetVersion(), + "app": "admission-webhook", + "openebs.io/component-name": "admission-webhook", + string(apis.OpenEBSVersionKey): version.GetVersion(), }, OwnerReferences: []metav1.OwnerReference{ownerReference}, }, @@ -423,20 +436,53 @@ func GetAdmissionReference() (*metav1.OwnerReference, error) { return nil, fmt.Errorf("failed to create deployment ownerReference") } +// addCSPCDeleteRule adds the DELETE operation to for CSPC if coming from 1.6.0 +// or older version +func addCSPCDeleteRule(config *v1beta1.ValidatingWebhookConfiguration) { + if config.Labels[string(apis.OpenEBSVersionKey)] < "1.7.0" { + index := -1 + // find the index of the RuleWithOperations having CSPC + for i, rule := range config.Webhooks[0].Rules { + if util.ContainsString(rule.Rule.Resources, "cstorpoolclusters") { + index = i + break + } + } + // if CSPC RuleWithOperations is found append DELETE operation + if index != -1 { + config.Webhooks[0].Rules[index].Operations = append( + config.Webhooks[0].Rules[index].Operations, + v1beta1.Delete, + ) + } + } +} + // preUpgrade checks for the required older webhook configs,older // then 1.4.0 if exists delete them. func preUpgrade(openebsNamespace string) error { - secretlist, err := secret.NewKubeClient(secret.WithNamespace(openebsNamespace)).List(metav1.ListOptions{LabelSelector: webhookLabel}) if err != nil { return fmt.Errorf("failed to list old secret: %s", err.Error()) } for _, scrt := range secretlist.Items { - if len(scrt.Labels["openebs.io/version"]) == 0 { - err = secret.NewKubeClient(secret.WithNamespace(openebsNamespace)).Delete(scrt.Name, &metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("failed to delete old secret %s: %s", scrt.Name, err.Error()) + if scrt.Labels[string(apis.OpenEBSVersionKey)] != version.Current() { + if scrt.Labels[string(apis.OpenEBSVersionKey)] == "" { + err = secret.NewKubeClient(secret.WithNamespace(openebsNamespace)).Delete(scrt.Name, &metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete old secret %s: %s", scrt.Name, err.Error()) + } + } else { + newScrt := scrt + for _, t := range transformSecret { + t(&newScrt) + } + newScrt.Labels[string(apis.OpenEBSVersionKey)] = version.Current() + _, err = secret.NewKubeClient(secret.WithNamespace(openebsNamespace)).Update(&newScrt) + if err != nil { + return fmt.Errorf("failed to update old secret %s: %s", scrt.Name, err.Error()) + } } } } @@ -447,10 +493,22 @@ func preUpgrade(openebsNamespace string) error { } for _, service := range svcList.Items { - if len(service.Labels["openebs.io/version"]) == 0 { - err = svc.NewKubeClient(svc.WithNamespace(openebsNamespace)).Delete(service.Name, &metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("failed to delete old service %s: %s", service.Name, err.Error()) + if service.Labels[string(apis.OpenEBSVersionKey)] != version.Current() { + if service.Labels[string(apis.OpenEBSVersionKey)] == "" { + err = svc.NewKubeClient(svc.WithNamespace(openebsNamespace)).Delete(service.Name, &metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete old service %s: %s", service.Name, err.Error()) + } + } else { + newSvc := service + for _, t := range transformSvc { + t(&newSvc) + } + newSvc.Labels[string(apis.OpenEBSVersionKey)] = version.Current() + _, err = svc.NewKubeClient(svc.WithNamespace(openebsNamespace)).Update(&newSvc) + if err != nil { + return fmt.Errorf("failed to update old service %s: %s", service.Name, err.Error()) + } } } } @@ -460,10 +518,22 @@ func preUpgrade(openebsNamespace string) error { } for _, config := range webhookConfigList.Items { - if len(config.Labels["openebs.io/version"]) == 0 { - err = validate.KubeClient().Delete(config.Name, &metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("failed to delete older webhook config %s: %s", config.Name, err.Error()) + if config.Labels[string(apis.OpenEBSVersionKey)] != version.Current() { + if config.Labels[string(apis.OpenEBSVersionKey)] == "" { + err = validate.KubeClient().Delete(config.Name, &metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete older webhook config %s: %s", config.Name, err.Error()) + } + } else { + newConfig := config + for _, t := range transformConfig { + t(&newConfig) + } + newConfig.Labels[string(apis.OpenEBSVersionKey)] = version.Current() + _, err = validate.KubeClient().Update(&newConfig) + if err != nil { + return fmt.Errorf("failed to update older webhook config %s: %s", config.Name, err.Error()) + } } } } diff --git a/pkg/webhook/cspc.go b/pkg/webhook/cspc.go index 5b10a513b6..6d415e2ce4 100644 --- a/pkg/webhook/cspc.go +++ b/pkg/webhook/cspc.go @@ -28,6 +28,8 @@ import ( blockdevice "github.com/openebs/maya/pkg/blockdevice/v1alpha2" blockdeviceclaim "github.com/openebs/maya/pkg/blockdeviceclaim/v1alpha1" cspcv1alpha1 "github.com/openebs/maya/pkg/cstor/poolcluster/v1alpha1" + cspi "github.com/openebs/maya/pkg/cstor/poolinstance/v1alpha3" + cvr "github.com/openebs/maya/pkg/cstor/volumereplica/v1alpha1" env "github.com/openebs/maya/pkg/env/v1alpha1" "github.com/pkg/errors" "k8s.io/api/admission/v1beta1" @@ -103,10 +105,11 @@ func (wh *webhook) validateCSPC(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionR } else if req.Operation == v1beta1.Create { klog.V(5).Infof("Admission webhook create request for type %s", req.Kind.Kind) return wh.validateCSPCCreateRequest(req) + } else if req.Operation == v1beta1.Delete { + klog.V(5).Infof("Admission webhook delete request for type %s", req.Kind.Kind) + return wh.validateCSPCDeleteRequest(req) } - klog.V(2).Info("Admission wehbook for PVC not " + - "configured for operations other than UPDATE and CREATE") return response } @@ -128,6 +131,38 @@ func (wh *webhook) validateCSPCCreateRequest(req *v1beta1.AdmissionRequest) *v1b return response } +// validateCSPCDeleteRequest validates CSPC delete request +// if any cvrs exist on the cspc pools then deletion is invalid +func (wh *webhook) validateCSPCDeleteRequest(req *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse { + response := NewAdmissionResponse().SetAllowed().WithResultAsSuccess(http.StatusAccepted).AR + cspiList, err := cspi.NewKubeClient().WithNamespace(req.Namespace).List( + metav1.ListOptions{ + LabelSelector: string(apis.CStorPoolClusterCPK) + "=" + req.Name, + }) + if err != nil { + klog.Errorf("Could not list cspi for cspc %s: %s", req.Name, err.Error()) + response = BuildForAPIObject(response).UnSetAllowed().WithResultAsFailure(err, http.StatusBadRequest).AR + return response + } + for _, cspiObj := range cspiList.Items { + // list cvrs in all namespaces + cvrList, err := cvr.NewKubeclient().WithNamespace("").List(metav1.ListOptions{ + LabelSelector: "cstorpoolinstance.openebs.io/name=" + cspiObj.Name, + }) + if err != nil { + klog.Errorf("Could not list cvr for cspi %s: %s", cspiObj.Name, err.Error()) + response = BuildForAPIObject(response).UnSetAllowed().WithResultAsFailure(err, http.StatusBadRequest).AR + return response + } + if len(cvrList.Items) != 0 { + err := errors.Errorf("invalid cspc %s deletion: volume still exists on pool %s", req.Name, cspiObj.Name) + response = BuildForAPIObject(response).UnSetAllowed().WithResultAsFailure(err, http.StatusUnprocessableEntity).AR + return response + } + } + return response +} + func cspcValidation(cspc *apis.CStorPoolCluster) (bool, string) { usedNodes := map[string]bool{} if len(cspc.Spec.Pools) == 0 {