[Feature] PVCResize action concurrency limit (#1378)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - (Maintenance) Bump golang.org/x/net to v0.13.0
+- (Feature) PVCResize action concurrency limit
 
 ## [1.2.32](https://github.com/arangodb/kube-arangodb/tree/1.2.32) (2023-08-07)
 - (Feature) Backup lifetime - remove Backup once its lifetime has been reached

internal/actions.yaml

@@ -106,6 +106,7 @@ actions:
   PVCResize:
     description: Start the resize procedure. Updates PVC Requests field
     timeout: 30m
+    configurable: true
   PVCResized:
     description: Waits for PVC resize to be completed
     timeout: 15m

pkg/deployment/reconcile/action.config.generated.go

@@ -23,9 +23,14 @@ import "github.com/spf13/cobra"
 var ActionsConfigGlobal ActionsConfig
 
 type ActionsConfig struct {
+	// PVCResize keeps configuration for action api.ActionTypePVCResize
+	PVCResize ActionPVCResizeConfig
 }
 
 // Init initializes all registered actions config options.
 func (a *ActionsConfig) Init(cmd *cobra.Command) error {
+	if err := a.PVCResize.Init(cmd, "action.PVCResize"); err != nil {
+		return err
+	}
 	return nil
 }

pkg/deployment/reconcile/action_pvc_resize.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,7 +22,9 @@ package reconcile
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/spf13/cobra"
 	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -32,6 +34,18 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
+type ActionPVCResizeConfig struct {
+	Concurrency int
+}
+
+func (a *ActionPVCResizeConfig) Init(cmd *cobra.Command, section string) error {
+	f := cmd.Flags()
+
+	f.IntVar(&a.Concurrency, fmt.Sprintf("%s.concurrency", section), 32, "Define limit of concurrent PVC Resizes on the cluster")
+
+	return nil
+}
+
 // newRotateMemberAction creates a new Action that implements the given
 // planned RotateMember action.
 func newPVCResizeAction(action api.Action, actionCtx ActionContext) Action {

pkg/deployment/reconcile/plan_builder_storage.go

@@ -47,12 +47,39 @@ func (r *Reconciler) createRotateServerStorageResizePlanRotate(ctx context.Conte
 func (r *Reconciler) createRotateServerStorageResizePlanInternal(spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext, mode api.PVCResizeMode) api.Plan {
 	var plan api.Plan
 
+	pvcs := map[string]*core.PersistentVolumeClaim{}
+
 	for _, member := range status.Members.AsList() {
 		cache, ok := context.ACS().ClusterCache(member.Member.ClusterID)
 		if !ok {
 			// Do not work without cache
 			continue
 		}
+
+		if member.Member.PersistentVolumeClaim.GetName() == "" {
+			// Plan is irrelevant without PVC
+			continue
+		}
+
+		pvc, exists := cache.PersistentVolumeClaim().V1().GetSimple(member.Member.PersistentVolumeClaim.GetName())
+		if exists {
+			pvcs[member.Member.ID] = pvc.DeepCopy()
+		}
+	}
+
+	resizing := 0
+
+	for _, pvc := range pvcs {
+		if k8sutil.IsPersistentVolumeClaimResizing(pvc) {
+			resizing++
+		}
+	}
+
+	for _, member := range status.Members.AsList() {
+		if resizing >= ActionsConfigGlobal.PVCResize.Concurrency {
+			break
+		}
+
 		if member.Member.Phase != api.MemberPhaseCreated {
 			// Only make changes when phase is created
 			continue
@@ -73,7 +100,7 @@ func (r *Reconciler) createRotateServerStorageResizePlanInternal(spec api.Deploy
 		}
 
 		// Load PVC
-		pvc, exists := cache.PersistentVolumeClaim().V1().GetSimple(member.Member.PersistentVolumeClaim.GetName())
+		pvc, exists := pvcs[member.Member.PersistentVolumeClaim.GetName()]
 		if !exists {
 			r.planLogger.
 				Str("role", member.Group.AsRole()).

pkg/util/k8sutil/pvc.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -48,6 +48,16 @@ func IsPersistentVolumeClaimFileSystemResizePending(pvc *core.PersistentVolumeCl
 	return false
 }
 
+// IsPersistentVolumeClaimResizing returns true if the pvc has Resizing set to true
+func IsPersistentVolumeClaimResizing(pvc *core.PersistentVolumeClaim) bool {
+	for _, c := range pvc.Status.Conditions {
+		if c.Type == core.PersistentVolumeClaimResizing && c.Status == core.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
 // ExtractStorageResourceRequirement filters resource requirements for Pods.
 func ExtractStorageResourceRequirement(resources core.ResourceRequirements) core.ResourceRequirements {