From f1b11f1c2bb3956b6749caf5b92883984251ba72 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 15 Aug 2023 11:47:23 +0000 Subject: [PATCH 1/2] [Feature] PVCResize action concurrency limit --- CHANGELOG.md | 1 + internal/actions.yaml | 1 + .../reconcile/action.config.generated.go | 5 ++++ pkg/deployment/reconcile/action_pvc_resize.go | 14 +++++++++ .../reconcile/plan_builder_storage.go | 29 ++++++++++++++++++- pkg/util/k8sutil/pvc.go | 10 +++++++ 6 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f83c3620..ca9a29bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) - (Maintenance) Bump golang.org/x/net to v0.13.0 +- (Feature) PVCResize action concurrency limit ## [1.2.32](https://github.com/arangodb/kube-arangodb/tree/1.2.32) (2023-08-07) - (Feature) Backup lifetime - remove Backup once its lifetime has been reached diff --git a/internal/actions.yaml b/internal/actions.yaml index 3825b826e..da6908374 100644 --- a/internal/actions.yaml +++ b/internal/actions.yaml @@ -106,6 +106,7 @@ actions: PVCResize: description: Start the resize procedure. Updates PVC Requests field timeout: 30m + configurable: true PVCResized: description: Waits for PVC resize to be completed timeout: 15m diff --git a/pkg/deployment/reconcile/action.config.generated.go b/pkg/deployment/reconcile/action.config.generated.go index 046a236c2..40e7e737a 100644 --- a/pkg/deployment/reconcile/action.config.generated.go +++ b/pkg/deployment/reconcile/action.config.generated.go @@ -23,9 +23,14 @@ import "github.com/spf13/cobra" var ActionsConfigGlobal ActionsConfig type ActionsConfig struct { + // PVCResize keeps configuration for action api.ActionTypePVCResize + PVCResize ActionPVCResizeConfig } // Init initializes all registered actions config options. func (a *ActionsConfig) Init(cmd *cobra.Command) error { + if err := a.PVCResize.Init(cmd, "action.PVCResize"); err != nil { + return err + } return nil } diff --git a/pkg/deployment/reconcile/action_pvc_resize.go b/pkg/deployment/reconcile/action_pvc_resize.go index 3d6e04d13..0877fe45d 100644 --- a/pkg/deployment/reconcile/action_pvc_resize.go +++ b/pkg/deployment/reconcile/action_pvc_resize.go @@ -22,7 +22,9 @@ package reconcile import ( "context" + "fmt" + "github.com/spf13/cobra" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,6 +34,18 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) +type ActionPVCResizeConfig struct { + Concurrency int +} + +func (a *ActionPVCResizeConfig) Init(cmd *cobra.Command, section string) error { + f := cmd.Flags() + + f.IntVar(&a.Concurrency, fmt.Sprintf("%s.concurrency", section), 32, "Define limit of concurrent PVC Resizes on the cluster") + + return nil +} + // newRotateMemberAction creates a new Action that implements the given // planned RotateMember action. func newPVCResizeAction(action api.Action, actionCtx ActionContext) Action { diff --git a/pkg/deployment/reconcile/plan_builder_storage.go b/pkg/deployment/reconcile/plan_builder_storage.go index a8974a0da..fc8b2d160 100644 --- a/pkg/deployment/reconcile/plan_builder_storage.go +++ b/pkg/deployment/reconcile/plan_builder_storage.go @@ -47,12 +47,39 @@ func (r *Reconciler) createRotateServerStorageResizePlanRotate(ctx context.Conte func (r *Reconciler) createRotateServerStorageResizePlanInternal(spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext, mode api.PVCResizeMode) api.Plan { var plan api.Plan + pvcs := map[string]*core.PersistentVolumeClaim{} + for _, member := range status.Members.AsList() { cache, ok := context.ACS().ClusterCache(member.Member.ClusterID) if !ok { // Do not work without cache continue } + + if member.Member.PersistentVolumeClaim.GetName() == "" { + // Plan is irrelevant without PVC + continue + } + + pvc, exists := cache.PersistentVolumeClaim().V1().GetSimple(member.Member.PersistentVolumeClaim.GetName()) + if exists { + pvcs[member.Member.ID] = pvc.DeepCopy() + } + } + + resizing := 0 + + for _, pvc := range pvcs { + if k8sutil.IsPersistentVolumeClaimResizing(pvc) { + resizing++ + } + } + + for _, member := range status.Members.AsList() { + if resizing >= ActionsConfigGlobal.PVCResize.Concurrency { + break + } + if member.Member.Phase != api.MemberPhaseCreated { // Only make changes when phase is created continue @@ -73,7 +100,7 @@ func (r *Reconciler) createRotateServerStorageResizePlanInternal(spec api.Deploy } // Load PVC - pvc, exists := cache.PersistentVolumeClaim().V1().GetSimple(member.Member.PersistentVolumeClaim.GetName()) + pvc, exists := pvcs[member.Member.PersistentVolumeClaim.GetName()] if !exists { r.planLogger. Str("role", member.Group.AsRole()). diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index a2715102c..f891fea18 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -48,6 +48,16 @@ func IsPersistentVolumeClaimFileSystemResizePending(pvc *core.PersistentVolumeCl return false } +// IsPersistentVolumeClaimResizing returns true if the pvc has Resizing set to true +func IsPersistentVolumeClaimResizing(pvc *core.PersistentVolumeClaim) bool { + for _, c := range pvc.Status.Conditions { + if c.Type == core.PersistentVolumeClaimResizing && c.Status == core.ConditionTrue { + return true + } + } + return false +} + // ExtractStorageResourceRequirement filters resource requirements for Pods. func ExtractStorageResourceRequirement(resources core.ResourceRequirements) core.ResourceRequirements { From e6a7720935b3d9483a916c20abd309bfddafdfdc Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 16 Aug 2023 07:04:37 +0000 Subject: [PATCH 2/2] Lint --- pkg/deployment/reconcile/action_pvc_resize.go | 2 +- pkg/util/k8sutil/pvc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/deployment/reconcile/action_pvc_resize.go b/pkg/deployment/reconcile/action_pvc_resize.go index 0877fe45d..386bd5d04 100644 --- a/pkg/deployment/reconcile/action_pvc_resize.go +++ b/pkg/deployment/reconcile/action_pvc_resize.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index f891fea18..d834df8dc 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.