From e99bc678d4149e5bb37c09df4c0a03fc946b3e82 Mon Sep 17 00:00:00 2001 From: Kasakaze Date: Mon, 9 Oct 2023 03:56:54 +0000 Subject: [PATCH 1/4] feat: support unclear stale pvc Signed-off-by: Kasakaze --- provisioner/env.go | 7 +++++++ provisioner/provisioner.go | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/provisioner/env.go b/provisioner/env.go index 9fcf767d..625ca981 100644 --- a/provisioner/env.go +++ b/provisioner/env.go @@ -58,6 +58,9 @@ const ( // NFSBackendPvcTimeout defines env name to store BackendPvcBoundTimeout value NFSBackendPvcTimeout menv.ENVKey = "OPENEBS_IO_NFS_SERVER_BACKEND_PVC_TIMEOUT" + // NFSCleanUpStalePvcEnable defines env name to store the value of clean stale pvc enable + NFSCleanUpStalePvcEnable menv.ENVKey = "OPENEBS_IO_NFS_SERVER_CLEANUP_STALE_PVC_ENABLE" + // NFSServerImagePullSecret defines the env name to store the name of the image pull secret NFSServerImagePullSecret menv.ENVKey = "OPENEBS_IO_NFS_SERVER_IMAGE_PULL_SECRET" ) @@ -105,6 +108,10 @@ func getBackendPvcTimeout() string { return menv.Get(NFSBackendPvcTimeout) } +// OPENEBS_IO_NFS_SERVER_CLEAN_STALE_PVC_ENABLE +func getNfsServerCleanUpStalePvcEnable() string { + return menv.Get(NFSCleanUpStalePvcEnable) +} func getNfsServerImagePullSecret() string { return menv.GetOrDefault(NFSServerImagePullSecret, "") } diff --git a/provisioner/provisioner.go b/provisioner/provisioner.go index b2dcd527..0e6e78be 100644 --- a/provisioner/provisioner.go +++ b/provisioner/provisioner.go @@ -116,8 +116,18 @@ func NewProvisioner(ctx context.Context, kubeClient *clientset.Clientset) (*Prov // and maintain it in cache go k8sNodeInformer.Run(ctx.Done()) - // Running garbage collector to perform cleanup for stale NFS resources - go RunGarbageCollector(ctx, kubeClient, pvTracker, nfsServerNs) + cleanUpStalePvcStr := getNfsServerCleanUpStalePvcEnable() + cleanUpStalePvcEnable, err := strconv.ParseBool(cleanUpStalePvcStr) + if err != nil { + klog.Warningf("Invalid cleanUpStalePvc value=%s, using default value=true", cleanUpStalePvcStr) + cleanUpStalePvcEnable = true + } + if cleanUpStalePvcEnable { + // Running garbage collector to perform cleanup for stale NFS resources + go RunGarbageCollector(ctx, kubeClient, pvTracker, nfsServerNs) + } else { + klog.Warningf("CleanUpStalePvc is disabled") + } return p, nil } From 02840d0738b0c8b591fe1733cb37d6f58270d24d Mon Sep 17 00:00:00 2001 From: Kasakaze Date: Mon, 20 Nov 2023 15:50:38 +0000 Subject: [PATCH 2/4] make test Signed-off-by: Kasakaze --- .../api/apps/v1/deployment/deployment.go | 6 +- .../v1/persistentvolume/persistentvolume.go | 16 +-- .../v1/podtemplatespec/podtemplatespec.go | 9 +- pkg/kubernetes/api/core/v1/volume/build.go | 4 +- pkg/kubernetes/client/client.go | 22 ++-- provisioner/config.go | 74 +++++++------ provisioner/config_test.go | 2 +- provisioner/node_affinity.go | 87 +++++++-------- provisioner/provisioner.go | 16 +-- provisioner/provisioner_kernel_nfs_server.go | 8 +- provisioner/signal_handler.go | 2 +- provisioner/types.go | 101 +++++++++--------- 12 files changed, 188 insertions(+), 159 deletions(-) diff --git a/pkg/kubernetes/api/apps/v1/deployment/deployment.go b/pkg/kubernetes/api/apps/v1/deployment/deployment.go index b96bc78f..70a6b933 100644 --- a/pkg/kubernetes/api/apps/v1/deployment/deployment.go +++ b/pkg/kubernetes/api/apps/v1/deployment/deployment.go @@ -307,7 +307,7 @@ func (b *Builder) WithReplicas(replicas *int32) *Builder { return b } -//WithStrategyType sets the strategy field of the deployment +// WithStrategyType sets the strategy field of the deployment func (b *Builder) WithStrategyType( strategytype appsv1.DeploymentStrategyType, ) *Builder { @@ -323,7 +323,7 @@ func (b *Builder) WithStrategyType( return b } -//WithStrategyTypeRecreate sets the strategy field of the deployment as Recreate +// WithStrategyTypeRecreate sets the strategy field of the deployment as Recreate func (b *Builder) WithStrategyTypeRecreate() *Builder { return b.WithStrategyType(appsv1.RecreateDeploymentStrategyType) } @@ -519,7 +519,7 @@ func (d *Deploy) IsTerminationInProgress() bool { // IsUpdateInProgress Checks if all the replicas are updated or not. // If Status.AvailableReplicas < Status.UpdatedReplicas then all the -//older replicas are not there but there are less number of availableReplicas +// older replicas are not there but there are less number of availableReplicas func IsUpdateInProgress() Predicate { return func(d *Deploy) bool { return d.IsUpdateInProgress() diff --git a/pkg/kubernetes/api/core/v1/persistentvolume/persistentvolume.go b/pkg/kubernetes/api/core/v1/persistentvolume/persistentvolume.go index 89aeefe5..592112a2 100644 --- a/pkg/kubernetes/api/core/v1/persistentvolume/persistentvolume.go +++ b/pkg/kubernetes/api/core/v1/persistentvolume/persistentvolume.go @@ -91,15 +91,15 @@ func (p *PV) GetPath() string { // This method expects only a single hostname to be set. // // The PV object will have the node's hostname specified as follows: -// nodeAffinity: -// required: -// nodeSelectorTerms: -// - matchExpressions: -// - key: kubernetes.io/hostname -// operator: In -// values: -// - hostname // +// nodeAffinity: +// required: +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/hostname +// operator: In +// values: +// - hostname func (p *PV) GetAffinitedNodeHostname() string { nodeAffinity := p.object.Spec.NodeAffinity if nodeAffinity == nil { diff --git a/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go b/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go index 647c378b..d657d8c4 100644 --- a/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go +++ b/pkg/kubernetes/api/core/v1/podtemplatespec/podtemplatespec.go @@ -225,7 +225,7 @@ func (b *Builder) WithNodeSelectorNew(nodeselectors map[string]string) *Builder return b } -//WithNodeSelectorByValue overrides the NodeSelector with new values +// WithNodeSelectorByValue overrides the NodeSelector with new values func (b *Builder) WithNodeSelectorByValue(nodeselectors map[string]string) *Builder { // copy of original map newnodeselectors := map[string]string{} @@ -290,9 +290,12 @@ func (b *Builder) WithAffinity(affinity *corev1.Affinity) *Builder { // WithNodeAffinityMatchExpressions sets matchexpressions under // nodeAffinity // NOTE: If nil is passed then match expressions will not be -// propogated to node affinity. +// +// propogated to node affinity. +// // CAUTION: Don't invoke WithAffinity func after calling this function -// It will overwrite MatchExpression +// +// It will overwrite MatchExpression func (b *Builder) WithNodeAffinityMatchExpressions( mExpressions []corev1.NodeSelectorRequirement) *Builder { if len(mExpressions) == 0 { diff --git a/pkg/kubernetes/api/core/v1/volume/build.go b/pkg/kubernetes/api/core/v1/volume/build.go index becfc7c8..60173be0 100644 --- a/pkg/kubernetes/api/core/v1/volume/build.go +++ b/pkg/kubernetes/api/core/v1/volume/build.go @@ -65,7 +65,7 @@ func (b *Builder) WithHostDirectory(path string) *Builder { return b } -//WithSecret sets the VolumeSource field of Volume with provided Secret +// WithSecret sets the VolumeSource field of Volume with provided Secret func (b *Builder) WithSecret(secret *corev1.Secret, defaultMode int32) *Builder { dM := defaultMode if secret == nil { @@ -92,7 +92,7 @@ func (b *Builder) WithSecret(secret *corev1.Secret, defaultMode int32) *Builder return b } -//WithConfigMap sets the VolumeSource field of Volume with provided ConfigMap +// WithConfigMap sets the VolumeSource field of Volume with provided ConfigMap func (b *Builder) WithConfigMap(configMap *corev1.ConfigMap, defaultMode int32) *Builder { dM := defaultMode if configMap == nil { diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go index 48ead57c..f6feacd5 100644 --- a/pkg/kubernetes/client/client.go +++ b/pkg/kubernetes/client/client.go @@ -42,7 +42,8 @@ const ( // to abstract getting kubernetes incluster config // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type getInClusterConfigFn func() (*rest.Config, error) // buildConfigFromFlagsFn is a typed function @@ -50,7 +51,8 @@ type getInClusterConfigFn func() (*rest.Config, error) // provided flags // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type buildConfigFromFlagsFn func(string, string) (*rest.Config, error) // getKubeMasterIPFromENVFn is a typed function @@ -58,7 +60,8 @@ type buildConfigFromFlagsFn func(string, string) (*rest.Config, error) // address from environment variable // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type getKubeMasterIPFromENVFn func(env.ENVKey) string // getKubeConfigPathFromENVFn is a typed function to @@ -66,21 +69,24 @@ type getKubeMasterIPFromENVFn func(env.ENVKey) string // environment variable // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type getKubeConfigPathFromENVFn func(env.ENVKey) string // getKubeDynamicClientFn is a typed function to // abstract getting dynamic kubernetes clientset // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type getKubeDynamicClientFn func(*rest.Config) (dynamic.Interface, error) // getKubeClientsetFn is a typed function // to abstract getting kubernetes clientset // // NOTE: -// typed function makes it simple to mock +// +// typed function makes it simple to mock type getKubeClientsetFn func(*rest.Config) (*kubernetes.Clientset, error) // Client provides Kubernetes client operations @@ -119,7 +125,9 @@ type Client struct { // instance // // NOTE: -// This is the basic building block to create +// +// This is the basic building block to create +// // functional operations against the client // instance type OptionFn func(*Client) diff --git a/provisioner/config.go b/provisioner/config.go index 5142ab91..e2cc5fc9 100644 --- a/provisioner/config.go +++ b/provisioner/config.go @@ -107,7 +107,7 @@ const ( betaStorageClassAnnotation = "volume.beta.kubernetes.io/storage-class" ) -//GetVolumeConfig creates a new VolumeConfig struct by +// GetVolumeConfig creates a new VolumeConfig struct by // parsing and merging the configuration provided in the PVC // annotation - cas.openebs.io/config with the // default configuration of the provisioner. @@ -177,7 +177,7 @@ func (p *Provisioner) GetVolumeConfig(pvName string, pvc *v1.PersistentVolumeCla return c, nil } -//GetNFSServerTypeFromConfig returns the NFSServerType value configured +// GetNFSServerTypeFromConfig returns the NFSServerType value configured // in StorageClass. Default is kernel func (c *VolumeConfig) GetNFSServerTypeFromConfig() string { serverType := c.getValue(KeyPVNFSServerType) @@ -187,7 +187,7 @@ func (c *VolumeConfig) GetNFSServerTypeFromConfig() string { return serverType } -//GetBackendStorageClassFromConfig returns the Storage Class +// GetBackendStorageClassFromConfig returns the Storage Class // value configured in StorageClass. Default is "" func (c *VolumeConfig) GetBackendStorageClassFromConfig() string { backingSC := c.getValue(KeyPVBackendStorageClass) @@ -241,14 +241,16 @@ func (c *VolumeConfig) GetNFServerGraceTime() (int, error) { // StorageClass if specified // ----------------------------------------------------- // NOTE: This feature has been deprecated -// Alternative: Use FilePermission 'cas.openebs.io/config' annotation -// key on the backend volume PVC. Sample FilePermissions -// for FSGID-like configuration -- // -// name: FilePermissions -// data: -// GID: -// mode: "g+s" +// Alternative: Use FilePermission 'cas.openebs.io/config' annotation +// key on the backend volume PVC. Sample FilePermissions +// for FSGID-like configuration -- +// +// name: FilePermissions +// data: +// GID: +// mode: "g+s" +// // ----------------------------------------------------- func (c *VolumeConfig) GetFSGroupID() (*int64, error) { fsGroupIDStr := c.getValue(FSGroupID) @@ -371,18 +373,21 @@ func (c *VolumeConfig) getResourceList(key string) (v1.ResourceList, error) { return resourceList, nil } -//getValue is a utility function to extract the value +// getValue is a utility function to extract the value // of the `key` from the ConfigMap object - which is // map[string]interface{map[string][string]} // Example: -// { -// key1: { -// value: value1 -// enabled: true -// } -// } +// +// { +// key1: { +// value: value1 +// enabled: true +// } +// } +// // In the above example, if `key1` is passed as input, -// `value1` will be returned. +// +// `value1` will be returned. func (c *VolumeConfig) getValue(key string) string { if configObj, ok := util.GetNestedField(c.options, key).(map[string]string); ok { if val, p := configObj[string(mconfig.ValuePTP)]; p { @@ -392,20 +397,23 @@ func (c *VolumeConfig) getValue(key string) string { return "" } -//getData is a utility function to extract the value +// getData is a utility function to extract the value // of the `key` from the ConfigMap object - which is // map[string]interface{map[string]interface{map[string]string}} // Example: -// { -// key1: { -// value: value1 -// data: { -// dataKey1: dataValue1 -// } -// } -// } +// +// { +// key1: { +// value: value1 +// data: { +// dataKey1: dataValue1 +// } +// } +// } +// // In the above example, if `key1` and `dataKey1` are passed as input, -// `dataValue1` will be returned. +// +// `dataValue1` will be returned. func (c *VolumeConfig) getData(key string, dataKey string) string { if configData, ok := util.GetNestedField(c.configData, key).(map[string]string); ok { if val, p := configData[dataKey]; p { @@ -449,11 +457,11 @@ func hookConfigFileExist() (bool, error) { // initializeHook read the hook config file and update the given hook variable // return value: -// - nil -// - If hook config file doesn't exists -// - If hook config file is parsed and given hook variable is updated -// - error -// - If hook config is invalid +// - nil +// - If hook config file doesn't exists +// - If hook config file is parsed and given hook variable is updated +// - error +// - If hook config is invalid func initializeHook(hook **nfshook.Hook) error { hookFileExists, err := hookConfigFileExist() if err != nil { diff --git a/provisioner/config_test.go b/provisioner/config_test.go index 9739bf21..d5f7f43c 100644 --- a/provisioner/config_test.go +++ b/provisioner/config_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/provisioner/node_affinity.go b/provisioner/node_affinity.go index 46b20d5c..37ee9af7 100644 --- a/provisioner/node_affinity.go +++ b/provisioner/node_affinity.go @@ -44,38 +44,39 @@ func getNodeAffinityRules() NodeAffinity { // getOneOrMoreNodeSelectorRequirements can take one or more node affinity requirements // as string and convert them to structured form of Requirements // Ex: -// Case1 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/zone:[zone-1,zone-2,zone-3] // -// Return value: -// - key: kubernetes.io/storage-node -// operator: Exists -// - key: kubernetes.io/nfs-node -// operator: Exists -// - key: kubernetes.io/zone -// operator: In -// values: -// - zone-1 -// - zone-2 -// - zone-3 +// Case1 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/zone:[zone-1,zone-2,zone-3] // -// Case2 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/linux-amd64 +// Return value: +// - key: kubernetes.io/storage-node +// operator: Exists +// - key: kubernetes.io/nfs-node +// operator: Exists +// - key: kubernetes.io/zone +// operator: In +// values: +// - zone-1 +// - zone-2 +// - zone-3 // -// Return value: -// - key: kubernetes.io/storage-node -// operator: Exists -// - key: kubernetes.io/nfs-node -// operator: Exists -// - key: kubernetes.io/linux-amd64 -// operator: Exists +// Case2 - Input argument: kubernetes.io/storage-node,kubernetes.io/nfs-node,kubernetes.io/linux-amd64 // -// Case3 - Input argument: kubernetes.io/zone:[zone-1,zone-2] +// Return value: +// - key: kubernetes.io/storage-node +// operator: Exists +// - key: kubernetes.io/nfs-node +// operator: Exists +// - key: kubernetes.io/linux-amd64 +// operator: Exists // -// Return value: -// - key: kubernetes.io/zone -// operator: In -// values: -// - zone-1 -// - zone-2 +// Case3 - Input argument: kubernetes.io/zone:[zone-1,zone-2] +// +// Return value: +// - key: kubernetes.io/zone +// operator: In +// values: +// - zone-1 +// - zone-2 func getOneOrMoreNodeSelectorRequirements( requirementsAsValue string) []corev1.NodeSelectorRequirement { var nodeRequirements []corev1.NodeSelectorRequirement @@ -120,25 +121,25 @@ func getOneOrMoreNodeSelectorRequirements( // // Example: kubernetes.io/hostName:[z1-host1,z2-host1,z3-host1] value convert as below // -// key: kubernetes.io/hostName -// operator: "In" -// values: -// - z1-host1 -// - z2-host1 -// - z3-host1 +// key: kubernetes.io/hostName +// operator: "In" +// values: +// - z1-host1 +// - z2-host1 +// - z3-host1 // // Example: kubernetes.io/hostName:[region-1,region-2 value convert as below // -// key: kubernetes.io/hostName -// operator: "In" -// values: -// - region-1 -// - region-2 +// key: kubernetes.io/hostName +// operator: "In" +// values: +// - region-1 +// - region-2 // // Example: kubernetes.io/storage-node // -// key: kubernetes.io/storage-node -// operator: "Exists" +// key: kubernetes.io/storage-node +// operator: "Exists" func getNodeSelectorRequirement(reqAsValue string) corev1.NodeSelectorRequirement { var nsRequirement corev1.NodeSelectorRequirement keyValues := strings.Split(reqAsValue, ":") @@ -169,9 +170,9 @@ func getNodeSelectorRequirement(reqAsValue string) corev1.NodeSelectorRequiremen // getRightMostMatchingString will return right must matching string // which satisfies given pattern // Example: -// - Fetch last pattern matching on string -// Pattern: {,.*:\[.*} string: "key1,key2,key3:[v1, v2, v3]" -// Return value: key3:[v1, v2, v3] +// - Fetch last pattern matching on string +// Pattern: {,.*:\[.*} string: "key1,key2,key3:[v1, v2, v3]" +// Return value: key3:[v1, v2, v3] func getRightMostMatchingString(regex *regexp.Regexp, value string) string { loc := regex.FindStringIndex(value) if len(loc) == 0 { diff --git a/provisioner/provisioner.go b/provisioner/provisioner.go index 0e6e78be..2e4147a5 100644 --- a/provisioner/provisioner.go +++ b/provisioner/provisioner.go @@ -63,7 +63,8 @@ var ( ) // NewProvisioner will create a new Provisioner object and initialize -// it with global information used across PV create and delete operations. +// +// it with global information used across PV create and delete operations. func NewProvisioner(ctx context.Context, kubeClient *clientset.Clientset) (*Provisioner, error) { namespace := getOpenEBSNamespace() @@ -133,13 +134,15 @@ func NewProvisioner(ctx context.Context, kubeClient *clientset.Clientset) (*Prov } // SupportsBlock will be used by controller to determine if block mode is -// supported by the host path provisioner. +// +// supported by the host path provisioner. func (p *Provisioner) SupportsBlock() bool { return false } // Provision is invoked by the PVC controller which expect the PV -// to be provisioned and a valid PV spec returned. +// +// to be provisioned and a valid PV spec returned. func (p *Provisioner) Provision(ctx context.Context, opts pvController.ProvisionOptions) (*v1.PersistentVolume, pvController.ProvisioningState, error) { pvc := opts.PVC @@ -201,9 +204,10 @@ func (p *Provisioner) Provision(ctx context.Context, opts pvController.Provision } // Delete is invoked by the PVC controller to perform clean-up -// activities before deleteing the PV object. If reclaim policy is -// set to not-retain, then this function will create a helper pod -// to delete the host path from the node. +// +// activities before deleteing the PV object. If reclaim policy is +// set to not-retain, then this function will create a helper pod +// to delete the host path from the node. func (p *Provisioner) Delete(ctx context.Context, pv *v1.PersistentVolume) (err error) { p.pvTracker.Add(pv.Name) defer p.pvTracker.Delete(pv.Name) diff --git a/provisioner/provisioner_kernel_nfs_server.go b/provisioner/provisioner_kernel_nfs_server.go index ba8a235a..7b2a591e 100644 --- a/provisioner/provisioner_kernel_nfs_server.go +++ b/provisioner/provisioner_kernel_nfs_server.go @@ -31,7 +31,8 @@ import ( ) // ProvisionKernalNFSServer is invoked by the Provisioner to create a NFS -// with kernel NFS server +// +// with kernel NFS server func (p *Provisioner) ProvisionKernalNFSServer(ctx context.Context, opts pvController.ProvisionOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, error) { var leaseTime, graceTime int var leaseErr, graceErr error @@ -174,8 +175,9 @@ func (p *Provisioner) ProvisionKernalNFSServer(ctx context.Context, opts pvContr } // DeleteKernalNFSServer is invoked by the PVC controller to perform clean-up -// activities before deleteing the PV object. If reclaim policy is -// set to not-retain, then this function will delete the associated BDC +// +// activities before deleteing the PV object. If reclaim policy is +// set to not-retain, then this function will delete the associated BDC func (p *Provisioner) DeleteKernalNFSServer(ctx context.Context, pv *v1.PersistentVolume) (err error) { defer func() { err = errors.Wrapf(err, "failed to delete volume %v", pv.Name) diff --git a/provisioner/signal_handler.go b/provisioner/signal_handler.go index a26c02de..a33483bd 100644 --- a/provisioner/signal_handler.go +++ b/provisioner/signal_handler.go @@ -25,7 +25,7 @@ import ( "k8s.io/klog/v2" ) -//RegisterShutdownChannel closes the channel when signaled for termination +// RegisterShutdownChannel closes the channel when signaled for termination func RegisterShutdownChannel(cancelFn context.CancelFunc) { sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) diff --git a/provisioner/types.go b/provisioner/types.go index e5c1c885..c804dcd4 100644 --- a/provisioner/types.go +++ b/provisioner/types.go @@ -27,7 +27,7 @@ import ( listerv1 "k8s.io/client-go/listers/core/v1" ) -//Provisioner struct has the configuration and utilities required +// Provisioner struct has the configuration and utilities required // across the different work-flows. type Provisioner struct { stopCh <-chan struct{} @@ -68,20 +68,21 @@ type Provisioner struct { hook *nfshook.Hook } -//VolumeConfig struct contains the merged configuration of the PVC +// VolumeConfig struct contains the merged configuration of the PVC // and the associated SC. The configuration is derived from the // annotation `cas.openebs.io/config`. The configuration will be // in the following json format: -// { -// Key1:{ -// enabled: true -// value: "string value" -// }, -// Key2:{ -// enabled: true -// value: "string value" -// }, -// } +// +// { +// Key1:{ +// enabled: true +// value: "string value" +// }, +// Key2:{ +// enabled: true +// value: "string value" +// }, +// } type VolumeConfig struct { pvName string pvcName string @@ -91,7 +92,8 @@ type VolumeConfig struct { } // GetVolumeConfigFn allows to plugin a custom function -// and makes it easy to unit test provisioner +// +// and makes it easy to unit test provisioner type GetVolumeConfigFn func(pvName string, pvc *corev1.PersistentVolumeClaim) (*VolumeConfig, error) // NodeAffinity represents group of node affinity scheduling @@ -99,53 +101,54 @@ type GetVolumeConfigFn func(pvName string, pvc *corev1.PersistentVolumeClaim) (* // not configured then matches to no object i.e NFS Server can // schedule on any node in a cluster. Configured values will be // propogated to deployment.spec.template.spec.affinity.nodeAffinity. -// requiredDuringSchedulingIgnoredDuringExecution +// +// requiredDuringSchedulingIgnoredDuringExecution // // Values are propagated via ENV(NodeAffinity) on NFS Provisioner. // Example: Following can be various options to specify NodeAffinity rules // -// Config 1: Configure across zones and also storage should be available -// Env Value: "kubernetes.io/hostName:[z1-host1,z2-host1,z3-host1],kubernetes.io/storage:[available]" +// Config 1: Configure across zones and also storage should be available +// Env Value: "kubernetes.io/hostName:[z1-host1,z2-host1,z3-host1],kubernetes.io/storage:[available]" // -// Config 1 will be propogated as shown below on NFS-Server deployment -// nodeSelectorTerms: -// - matchExpressions: -// - key: kubernetes.io/hostName -// operator: "In" -// values: -// - z1-host1 -// - z2-host2 -// - z3-host3 -// - key: kubernetes.io/storage -// operator: "In" -// values: -// - available +// Config 1 will be propogated as shown below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/hostName +// operator: "In" +// values: +// - z1-host1 +// - z2-host2 +// - z3-host3 +// - key: kubernetes.io/storage +// operator: "In" +// values: +// - available // -// Config2: Configure on storage nodes in zone1 -// Env Value: "kubernetes.io/storage:[],kubernetes.io/zone:[zone1]" +// Config2: Configure on storage nodes in zone1 +// Env Value: "kubernetes.io/storage:[],kubernetes.io/zone:[zone1]" // -// Config2 will be propogated as shown below on NFS-Server deployment -// nodeSelectorTerms: -// - matchExpressions: -// - key: kubernetes.io/storage -// operator: "Exists" -// - key: kubernetes.io/zone -// operator: "In" -// values: -// - zone1 +// Config2 will be propogated as shown below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/storage +// operator: "Exists" +// - key: kubernetes.io/zone +// operator: "In" +// values: +// - zone1 // // -// Configi3: Configure on any storage node -// Env Value: "kubernetes.io/storage:[]" +// Configi3: Configure on any storage node +// Env Value: "kubernetes.io/storage:[]" // -// Config3 will be propogated as below on NFS-Server deployment -// nodeSelectorTerms: -// - matchExpressions: -// - key: kubernetes.io/storage -// operator: "Exists" +// Config3 will be propogated as below on NFS-Server deployment +// nodeSelectorTerms: +// - matchExpressions: +// - key: kubernetes.io/storage +// operator: "Exists" // -// Like shown above various combinations can be specified and before -// provisioning configuration will be validated +// Like shown above various combinations can be specified and before +// provisioning configuration will be validated // // NOTE: All the comma separated specification will be ANDed type NodeAffinity struct { From 65cb5c8c73542965ebcca1b2ab8b9f8360339832 Mon Sep 17 00:00:00 2001 From: Kasakaze Date: Mon, 20 Nov 2023 16:33:34 +0000 Subject: [PATCH 3/4] update: change CLEANUP_STALE_PVC_ENABLE to GARBAGE_COLLECTION_ENABLED Signed-off-by: Kasakaze --- provisioner/env.go | 9 ++++----- provisioner/provisioner.go | 12 ++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/provisioner/env.go b/provisioner/env.go index 625ca981..8ca932ac 100644 --- a/provisioner/env.go +++ b/provisioner/env.go @@ -58,8 +58,8 @@ const ( // NFSBackendPvcTimeout defines env name to store BackendPvcBoundTimeout value NFSBackendPvcTimeout menv.ENVKey = "OPENEBS_IO_NFS_SERVER_BACKEND_PVC_TIMEOUT" - // NFSCleanUpStalePvcEnable defines env name to store the value of clean stale pvc enable - NFSCleanUpStalePvcEnable menv.ENVKey = "OPENEBS_IO_NFS_SERVER_CLEANUP_STALE_PVC_ENABLE" + // The NFSGarbageCollectionEnable environment variable is the switch for the garbage collector.(default true) + NFSGarbageCollectionEnable menv.ENVKey = "OPENEBS_IO_NFS_SERVER_GARBAGE_COLLECTION_ENABLED" // NFSServerImagePullSecret defines the env name to store the name of the image pull secret NFSServerImagePullSecret menv.ENVKey = "OPENEBS_IO_NFS_SERVER_IMAGE_PULL_SECRET" @@ -108,9 +108,8 @@ func getBackendPvcTimeout() string { return menv.Get(NFSBackendPvcTimeout) } -// OPENEBS_IO_NFS_SERVER_CLEAN_STALE_PVC_ENABLE -func getNfsServerCleanUpStalePvcEnable() string { - return menv.Get(NFSCleanUpStalePvcEnable) +func getNfsGarbageCollectionEnable() string { + return menv.GetOrDefault(NFSGarbageCollectionEnable, "true") } func getNfsServerImagePullSecret() string { return menv.GetOrDefault(NFSServerImagePullSecret, "") diff --git a/provisioner/provisioner.go b/provisioner/provisioner.go index 2e4147a5..f3884be4 100644 --- a/provisioner/provisioner.go +++ b/provisioner/provisioner.go @@ -117,17 +117,17 @@ func NewProvisioner(ctx context.Context, kubeClient *clientset.Clientset) (*Prov // and maintain it in cache go k8sNodeInformer.Run(ctx.Done()) - cleanUpStalePvcStr := getNfsServerCleanUpStalePvcEnable() - cleanUpStalePvcEnable, err := strconv.ParseBool(cleanUpStalePvcStr) + gcStr := getNfsGarbageCollectionEnable() + gcEnable, err := strconv.ParseBool(gcStr) if err != nil { - klog.Warningf("Invalid cleanUpStalePvc value=%s, using default value=true", cleanUpStalePvcStr) - cleanUpStalePvcEnable = true + klog.Warningf("Invalid %s value=%s, using default value=true", NFSGarbageCollectionEnable, gcStr) + gcEnable = true } - if cleanUpStalePvcEnable { + if gcEnable { // Running garbage collector to perform cleanup for stale NFS resources go RunGarbageCollector(ctx, kubeClient, pvTracker, nfsServerNs) } else { - klog.Warningf("CleanUpStalePvc is disabled") + klog.Warning("Garbage collector is disabled") } return p, nil From 43943fa498bed55882ccc4cd7af53b23d55e54b1 Mon Sep 17 00:00:00 2001 From: Kasakaze Date: Wed, 22 Nov 2023 06:33:21 +0000 Subject: [PATCH 4/4] update: add env comments to kubectl deploy Signed-off-by: Kasakaze --- deploy/kubectl/openebs-nfs-provisioner.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/kubectl/openebs-nfs-provisioner.yaml b/deploy/kubectl/openebs-nfs-provisioner.yaml index 485c232c..6f88a522 100644 --- a/deploy/kubectl/openebs-nfs-provisioner.yaml +++ b/deploy/kubectl/openebs-nfs-provisioner.yaml @@ -112,6 +112,10 @@ spec: # value: "kubernetes.io/storage-node,kubernetes.io/nfs-node" # - name: OPENEBS_IO_NFS_SERVER_NODE_AFFINITY # value: "kubernetes.io/storage-node,kubernetes.io/nfs-node" + # Provide a switch to turn off the function of clearing stale pvc to avoid + #. garbage collecting an NFS backend PVC if the NFS PVC is deleted. + # - name: OPENEBS_IO_NFS_SERVER_GARBAGE_COLLECTION_ENABLED + #. value: false - name: NODE_NAME valueFrom: fieldRef: