From acccc7991aea0488bfc7f63a1159e3fda32f452a Mon Sep 17 00:00:00 2001 From: RealAnna Date: Thu, 6 Oct 2022 11:30:30 +0200 Subject: [PATCH 1/9] chore: Recursively check for pod status in permit plugin Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 9cf7cc21b5..48327ba428 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -38,7 +38,8 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 case Wait: klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on", p.GetObjectMeta().GetName()) - return framework.NewStatus(framework.Wait), 30 * time.Second + go pl.monitorPod(ctx, p) + return framework.NewStatus(framework.Wait), 60 * time.Second case Failure: klog.Infof("[Keptn Permit Plugin] failed pre-deployment checks on", p.GetObjectMeta().GetName()) return framework.NewStatus(framework.Error), 0 * time.Second @@ -52,6 +53,22 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 } +func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { + waitingPodHandler := pl.handler.GetWaitingPod(p.UID) + switch pl.workloadManager.Permit(ctx, p) { + case Failure: + klog.Infof("[Keptn Permit Plugin] pre-deployment checks failed for", p.GetObjectMeta().GetName()) + waitingPodHandler.Reject(Name, "Pre Deployment Check failed") + case Success: + klog.Infof("[Keptn Permit Plugin] pre-deployment checks passed for", p.GetObjectMeta().GetName()) + waitingPodHandler.Allow(Name) + default: + klog.Infof("[Keptn Permit Plugin] waiting pre-deployment checks for", p.GetObjectMeta().GetName()) + pl.monitorPod(ctx, p) + } + +} + // New initializes a new plugin and returns it. func New(_ runtime.Object, h framework.Handle) (framework.Plugin, error) { client, err := newClient() From 6f7dcf12aac256dadebf0d23d91431fc5f22b748 Mon Sep 17 00:00:00 2001 From: RealAnna Date: Thu, 6 Oct 2022 12:03:18 +0200 Subject: [PATCH 2/9] chore: Recursively check for pod status in permit plugin Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 48327ba428..ff3f35f049 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -64,6 +64,7 @@ func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { waitingPodHandler.Allow(Name) default: klog.Infof("[Keptn Permit Plugin] waiting pre-deployment checks for", p.GetObjectMeta().GetName()) + time.Sleep(5 * time.Second) pl.monitorPod(ctx, p) } From 16dd3ec235c6c3be79bc4c03f5aa943e1dbfad96 Mon Sep 17 00:00:00 2001 From: RealAnna Date: Fri, 7 Oct 2022 10:55:51 +0200 Subject: [PATCH 3/9] chore: Added rbac for configmap listing Signed-off-by: RealAnna --- scheduler/manifests/install/base/rbac.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scheduler/manifests/install/base/rbac.yaml b/scheduler/manifests/install/base/rbac.yaml index 7f5a098004..c72eae11c1 100644 --- a/scheduler/manifests/install/base/rbac.yaml +++ b/scheduler/manifests/install/base/rbac.yaml @@ -69,6 +69,9 @@ rules: - apiGroups: ["lifecycle.keptn.sh"] resources: ["keptnworkloadinstances"] verbs: ["get", "list", "watch"] + - apiGroups: [ "" ] + resources: [ "configmaps" ] + verbs: [ "list", "watch" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 From 40bf691178464146a04c4ed08eea4e7f155647b5 Mon Sep 17 00:00:00 2001 From: RealAnna Date: Fri, 7 Oct 2022 11:15:06 +0200 Subject: [PATCH 4/9] chore: Clean up Signed-off-by: RealAnna --- scheduler/cmd/scheduler/main.go | 2 +- scheduler/pkg/klcpermit/permit.go | 39 ++++++++++--------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/scheduler/cmd/scheduler/main.go b/scheduler/cmd/scheduler/main.go index 700bae1f8b..fe3a61bb06 100644 --- a/scheduler/cmd/scheduler/main.go +++ b/scheduler/cmd/scheduler/main.go @@ -29,7 +29,7 @@ func main() { rand.Seed(time.Now().UnixNano()) command := app.NewSchedulerCommand( - app.WithPlugin(klcpermit.Name, klcpermit.New), + app.WithPlugin(klcpermit.PluginName, klcpermit.New), ) code := cli.Run(command) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index ff3f35f049..1d51476a66 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -12,9 +12,9 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework" ) -// Name is the name of the plugin used in the plugin registry and configurations. +// PluginName is the name of the plugin used in the plugin registry and configurations. const ( - Name = "KLCPermit" + PluginName = "KLCPermit" ) // Permit is a plugin that implements a wait for pre-deployment checks @@ -25,31 +25,19 @@ type Permit struct { var _ framework.PermitPlugin = &Permit{} -// Name returns name of the plugin. +// PluginName returns name of the plugin. func (pl *Permit) Name() string { - return Name + return PluginName } func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (*framework.Status, time.Duration) { - klog.InfoS("[Keptn Permit Plugin] waiting for pre-deployment checks on", p.GetObjectMeta().GetName()) + klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on", p.GetObjectMeta().GetName()) + //start async CRD monitoring + go pl.monitorPod(ctx, p) - switch pl.workloadManager.Permit(ctx, p) { - - case Wait: - klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on", p.GetObjectMeta().GetName()) - go pl.monitorPod(ctx, p) - return framework.NewStatus(framework.Wait), 60 * time.Second - case Failure: - klog.Infof("[Keptn Permit Plugin] failed pre-deployment checks on", p.GetObjectMeta().GetName()) - return framework.NewStatus(framework.Error), 0 * time.Second - case Success: - klog.Infof("[Keptn Permit Plugin] passed pre-deployment checks on", p.GetObjectMeta().GetName()) - return framework.NewStatus(framework.Success), 0 * time.Second - default: - klog.Infof("[Keptn Permit Plugin] unknown status of pre-deployment checks for", p.GetObjectMeta().GetName()) - return framework.NewStatus(framework.Wait), 30 * time.Second - } + //queue pod and set eviction deadline + return framework.NewStatus(framework.Wait), 5 * time.Minute } @@ -57,14 +45,11 @@ func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { waitingPodHandler := pl.handler.GetWaitingPod(p.UID) switch pl.workloadManager.Permit(ctx, p) { case Failure: - klog.Infof("[Keptn Permit Plugin] pre-deployment checks failed for", p.GetObjectMeta().GetName()) - waitingPodHandler.Reject(Name, "Pre Deployment Check failed") + waitingPodHandler.Reject(PluginName, "Pre Deployment Check failed") case Success: - klog.Infof("[Keptn Permit Plugin] pre-deployment checks passed for", p.GetObjectMeta().GetName()) - waitingPodHandler.Allow(Name) + waitingPodHandler.Allow(PluginName) default: - klog.Infof("[Keptn Permit Plugin] waiting pre-deployment checks for", p.GetObjectMeta().GetName()) - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) pl.monitorPod(ctx, p) } From cc88a5f75941f7a3ac404b60970719075ec0dcbc Mon Sep 17 00:00:00 2001 From: RealAnna Date: Fri, 7 Oct 2022 11:27:25 +0200 Subject: [PATCH 5/9] chore: Clean up Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 1d51476a66..193c92a8e5 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -17,7 +17,7 @@ const ( PluginName = "KLCPermit" ) -// Permit is a plugin that implements a wait for pre-deployment checks +// Permit is a plugin that waits for pre-deployment checks type Permit struct { handler framework.Handle workloadManager *WorkloadManager @@ -32,10 +32,10 @@ func (pl *Permit) Name() string { func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1.Pod, nodeName string) (*framework.Status, time.Duration) { - klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on", p.GetObjectMeta().GetName()) + klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on %s", p.GetObjectMeta().GetName()) + //start async CRD monitoring go pl.monitorPod(ctx, p) - //queue pod and set eviction deadline return framework.NewStatus(framework.Wait), 5 * time.Minute From c605c5aef3668d9e778aad28e00188bc005d64f4 Mon Sep 17 00:00:00 2001 From: RealAnna Date: Fri, 7 Oct 2022 13:03:47 +0200 Subject: [PATCH 6/9] chore: Clean up was a failure :D Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 193c92a8e5..0423cf0388 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -34,15 +34,29 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on %s", p.GetObjectMeta().GetName()) - //start async CRD monitoring - go pl.monitorPod(ctx, p) - //queue pod and set eviction deadline - return framework.NewStatus(framework.Wait), 5 * time.Minute + // check the permit immediately, to fail early in case the pod cannot be queued + switch pl.workloadManager.Permit(ctx, p) { + + case Wait: + klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on %s", p.GetObjectMeta().GetName()) + go pl.monitorPod(ctx, p) + return framework.NewStatus(framework.Wait), 5 * time.Minute + case Failure: + klog.Infof("[Keptn Permit Plugin] failed pre-deployment checks on %s", p.GetObjectMeta().GetName()) + return framework.NewStatus(framework.Error), 0 * time.Second + case Success: + klog.Infof("[Keptn Permit Plugin] passed pre-deployment checks on %s", p.GetObjectMeta().GetName()) + return framework.NewStatus(framework.Success), 0 * time.Second + default: + klog.Infof("[Keptn Permit Plugin] unknown status of pre-deployment checks for %s", p.GetObjectMeta().GetName()) + return framework.NewStatus(framework.Wait), 5 * time.Minute + } } func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { waitingPodHandler := pl.handler.GetWaitingPod(p.UID) + switch pl.workloadManager.Permit(ctx, p) { case Failure: waitingPodHandler.Reject(PluginName, "Pre Deployment Check failed") @@ -52,7 +66,6 @@ func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { time.Sleep(10 * time.Second) pl.monitorPod(ctx, p) } - } // New initializes a new plugin and returns it. From b4a635cc2fad1eb01f3118ac99760668546a605f Mon Sep 17 00:00:00 2001 From: RealAnna Date: Fri, 7 Oct 2022 13:19:30 +0200 Subject: [PATCH 7/9] chore: Alwais check CRD status Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 0423cf0388..a73347754f 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -37,10 +37,6 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 // check the permit immediately, to fail early in case the pod cannot be queued switch pl.workloadManager.Permit(ctx, p) { - case Wait: - klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on %s", p.GetObjectMeta().GetName()) - go pl.monitorPod(ctx, p) - return framework.NewStatus(framework.Wait), 5 * time.Minute case Failure: klog.Infof("[Keptn Permit Plugin] failed pre-deployment checks on %s", p.GetObjectMeta().GetName()) return framework.NewStatus(framework.Error), 0 * time.Second @@ -48,7 +44,8 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 klog.Infof("[Keptn Permit Plugin] passed pre-deployment checks on %s", p.GetObjectMeta().GetName()) return framework.NewStatus(framework.Success), 0 * time.Second default: - klog.Infof("[Keptn Permit Plugin] unknown status of pre-deployment checks for %s", p.GetObjectMeta().GetName()) + klog.Infof("[Keptn Permit Plugin] waiting for pre-deployment checks on %s", p.GetObjectMeta().GetName()) + go pl.monitorPod(ctx, p) return framework.NewStatus(framework.Wait), 5 * time.Minute } From 17edbea2489fbde3607349521ade138b9ed2cd4a Mon Sep 17 00:00:00 2001 From: RealAnna Date: Mon, 10 Oct 2022 10:20:07 +0200 Subject: [PATCH 8/9] chore: from recursion to iterative loop Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index a73347754f..863bb22ba4 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -54,15 +54,19 @@ func (pl *Permit) Permit(ctx context.Context, state *framework.CycleState, p *v1 func (pl *Permit) monitorPod(ctx context.Context, p *v1.Pod) { waitingPodHandler := pl.handler.GetWaitingPod(p.UID) - switch pl.workloadManager.Permit(ctx, p) { - case Failure: - waitingPodHandler.Reject(PluginName, "Pre Deployment Check failed") - case Success: - waitingPodHandler.Allow(PluginName) - default: - time.Sleep(10 * time.Second) - pl.monitorPod(ctx, p) + for { + switch pl.workloadManager.Permit(ctx, p) { + case Failure: + waitingPodHandler.Reject(PluginName, "Pre Deployment Check failed") + return + case Success: + waitingPodHandler.Allow(PluginName) + return + default: + time.Sleep(10 * time.Second) + } } + } // New initializes a new plugin and returns it. From 6b88691c21b64f99fd24d885e86224c46be2a192 Mon Sep 17 00:00:00 2001 From: RealAnna Date: Mon, 10 Oct 2022 10:22:23 +0200 Subject: [PATCH 9/9] chore: according to review Signed-off-by: RealAnna --- scheduler/pkg/klcpermit/permit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler/pkg/klcpermit/permit.go b/scheduler/pkg/klcpermit/permit.go index 863bb22ba4..a6585479da 100644 --- a/scheduler/pkg/klcpermit/permit.go +++ b/scheduler/pkg/klcpermit/permit.go @@ -17,7 +17,7 @@ const ( PluginName = "KLCPermit" ) -// Permit is a plugin that waits for pre-deployment checks +// Permit is a plugin that waits for pre-deployment checks to be successfully finished type Permit struct { handler framework.Handle workloadManager *WorkloadManager