add poolcoordinator controller and webhooks to replace nodelifecycle …

…controller (openyurtio#1040) make yurt-controller-manager take care of webhook configurations and certs add unit tests
gnunu · Jan 15, 2023 · f54d6af · f54d6af
1 parent a1594b6
commit f54d6af
Show file tree

Hide file tree

Showing 34 changed files with 2,392 additions and 8,044 deletions.
diff --git a/charts/openyurt/templates/_helpers.tpl b/charts/openyurt/templates/_helpers.tpl
@@ -1 +1,28 @@
 {{/* vim: set filetype=mustache: */}}
+
+{{- define "yurt-controller-manager.fullname" -}}
+yurt-controller-manager
+{{- end -}}
+
+{{- define "yurt-controller-manager.name" -}}
+yurt-controller-manager
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "yurt-controller-manager.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "yurt-controller-manager.name" . }}
+app.kubernetes.io/instance: {{ printf "yurt-controller-manager-%s" .Release.Name }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "yurt-controller-manager.labels" -}}
+{{ include "yurt-controller-manager.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
diff --git a/charts/openyurt/templates/yurt-controller-manager.yaml b/charts/openyurt/templates/yurt-controller-manager.yaml
@@ -16,19 +16,9 @@ rules:
     resources:
       - nodes
     verbs:
-      - delete
       - get
       - list
-      - patch
-      - update
       - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
-      - update
   - apiGroups:
       - ""
     resources:
@@ -40,8 +30,12 @@ rules:
     resources:
       - pods
     verbs:
+      - create
       - delete
+      - get
       - list
+      - patch
+      - update
       - watch
   - apiGroups:
       - ""
@@ -129,6 +123,24 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
+    verbs:
+      - create
+      - delete
+      - get
+      - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -147,15 +159,18 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: yurt-controller-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "yurt-controller-manager.labels" . | nindent 4 }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: yurt-controller-manager
+      {{- include "yurt-controller-manager.selectorLabels" . | nindent 6 }}
   template:
     metadata:
       labels:
-        app: yurt-controller-manager
+        {{- include "yurt-controller-manager.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName: yurt-controller-manager
       hostNetwork: true
@@ -177,7 +192,53 @@ spec:
         image: "{{ .Values.yurtControllerManager.image.registry }}/{{ .Values.yurtControllerManager.image.repository }}:{{ .Values.yurtControllerManager.image.tag }}"
         imagePullPolicy: {{ .Values.yurtControllerManager.image.pullPolicy }}
         command:
-        - yurt-controller-manager
-      {{- if .Values.imagePullSecrets }}
-      imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
-      {{- end }}
+          - yurt-controller-manager
+        {{- if .Values.imagePullSecrets }}
+        imagePullSecrets:
+          {{ toYaml .Values.imagePullSecrets | nindent 10 }}
+        {{- end }}
+        ports:
+          - name: webhook-server
+            containerPort: {{ .Values.admissionWebhooks.service.port }}
+            protocol: TCP
+          - name: health
+            containerPort: 8000
+            protocol: TCP
+        env:
+          - name: WEBHOOK_CERT_DIR
+            value: {{ .Values.admissionWebhooks.certificate.mountPath }}
+          - name: WEBHOOK_SERVICE_PORT
+            value: {{ .Values.admissionWebhooks.service.port | quote }}
+          - name: WEBHOOK_SERVICE_NAME
+            value: {{ template "yurt-controller-manager.fullname" . }}-webhook
+          - name: WEBHOOK_NAMESPACE
+            value: {{ .Release.Namespace }}
+          - name: WEBHOOK_POD_VALIDATING_CONFIGURATION_NAME
+            value: {{ template "yurt-controller-manager.fullname" . }}
+          - name: WEBHOOK_POD_MUTATING_CONFIGURATION_NAME
+            value: {{ template "yurt-controller-manager.fullname" . }}
+          - name: WEBHOOK_POD_VALIDATING_NAME
+            value: {{ .Values.admissionWebhooks.names.validatingWebhookName }}
+          - name: WEBHOOK_POD_MUTATING_NAME
+            value: {{ .Values.admissionWebhooks.names.mutatingWebhookName }}
+          - name: WEBHOOK_POD_VALIDATING_PATH
+            value: {{ .Values.admissionWebhooks.names.webhookPodValidatingPath }}
+          - name: WEBHOOK_POD_MUTATING_PATH
+            value: {{ .Values.admissionWebhooks.names.webhookPodMutatingPath }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "yurt-controller-manager.name" . }}-webhook
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "yurt-controller-manager.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.admissionWebhooks.service.type }}
+  ports:
+    - port: 443
+      targetPort: {{ .Values.admissionWebhooks.service.port }}
+      protocol: TCP
+      name: https
+  selector:
+      {{ include "yurt-controller-manager.selectorLabels" . | nindent 6 }}
diff --git a/charts/openyurt/values.yaml b/charts/openyurt/values.yaml
@@ -13,6 +13,19 @@ yurtControllerManager:
     type: ClusterIP
     port: 80
 
+admissionWebhooks:
+  service:
+    type: ClusterIP
+    port: 9443
+  failurePolicy: Fail
+  certificate:
+    mountPath: /tmp/k8s-webhook-server/serving-certs
+  names:
+    validatingWebhookName: vpoolcoordinator.openyurt.io
+    mutatingWebhookName: mpoolcoordinator.openyurt.io
+    webhookPodValidatingPath: /pool-coordinator-webhook-validate
+    webhookPodMutatingPath: /pool-coordinator-webhook-mutate
+
 yurtTunnelAgent:
   replicaCount: 1
   tolerations: []

diff --git a/cmd/yurt-controller-manager/app/controllermanager.go b/cmd/yurt-controller-manager/app/controllermanager.go
@@ -309,10 +309,11 @@ var ControllersDisabledByDefault = sets.NewString()
 // paired to their InitFunc.  This allows for structured downstream composition and subdivision.
 func NewControllerInitializers() map[string]InitFunc {
 	controllers := map[string]InitFunc{}
-	controllers["nodelifecycle"] = startNodeLifecycleController
+	controllers["poolcoordinator"] = startPoolCoordinatorController
 	controllers["yurtcsrapprover"] = startYurtCSRApproverController
 	controllers["daemonpodupdater"] = startDaemonPodUpdaterController
 	controllers["servicetopologycontroller"] = startServiceTopologyController
+	controllers["webhookmanager"] = startWebhookManager
 	return controllers
 }
 

diff --git a/cmd/yurt-controller-manager/app/core.go b/cmd/yurt-controller-manager/app/core.go
@@ -23,37 +23,20 @@ package app
 
 import (
 	"net/http"
-	"time"
 
 	"github.com/openyurtio/openyurt/pkg/controller/certificates"
 	daemonpodupdater "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater"
-	lifecyclecontroller "github.com/openyurtio/openyurt/pkg/controller/nodelifecycle"
+	poolcoordinator "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator"
 	"github.com/openyurtio/openyurt/pkg/controller/servicetopology"
+	"github.com/openyurtio/openyurt/pkg/webhook"
 )
 
-func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, error) {
-	lifecycleController, err := lifecyclecontroller.NewNodeLifecycleController(
-		ctx.InformerFactory.Coordination().V1().Leases(),
-		ctx.InformerFactory.Core().V1().Pods(),
-		ctx.InformerFactory.Core().V1().Nodes(),
-		ctx.InformerFactory.Apps().V1().DaemonSets(),
-		// node lifecycle controller uses existing cluster role from node-controller
-		ctx.ClientBuilder.ClientOrDie("node-controller"),
-		//ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
-		5*time.Second,
-		ctx.ComponentConfig.NodeLifecycleController.NodeStartupGracePeriod.Duration,
-		ctx.ComponentConfig.NodeLifecycleController.NodeMonitorGracePeriod.Duration,
-		ctx.ComponentConfig.NodeLifecycleController.PodEvictionTimeout.Duration,
-		ctx.ComponentConfig.NodeLifecycleController.NodeEvictionRate,
-		ctx.ComponentConfig.NodeLifecycleController.SecondaryNodeEvictionRate,
-		ctx.ComponentConfig.NodeLifecycleController.LargeClusterSizeThreshold,
-		ctx.ComponentConfig.NodeLifecycleController.UnhealthyZoneThreshold,
-		*ctx.ComponentConfig.NodeLifecycleController.EnableTaintManager,
+func startPoolCoordinatorController(ctx ControllerContext) (http.Handler, bool, error) {
+	poolcoordinatorController := poolcoordinator.NewController(
+		ctx.ClientBuilder.ClientOrDie("poolcoordinator-controller"),
+		ctx.InformerFactory,
 	)
-	if err != nil {
-		return nil, true, err
-	}
-	go lifecycleController.Run(ctx.Stop)
+	go poolcoordinatorController.Run(ctx.Stop)
 	return nil, true, nil
 }
 
@@ -94,3 +77,12 @@ func startServiceTopologyController(ctx ControllerContext) (http.Handler, bool,
 	go svcTopologyController.Run(ctx.Stop)
 	return nil, true, nil
 }
+
+func startWebhookManager(ctx ControllerContext) (http.Handler, bool, error) {
+	webhookManager := webhook.NewWebhookManager(
+		ctx.ClientBuilder.ClientOrDie("webhook manager"),
+		ctx.InformerFactory,
+	)
+	go webhookManager.Run(ctx.Stop)
+	return nil, true, nil
+}
diff --git a/go.mod b/go.mod
@@ -26,6 +26,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
 	github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb
+	github.com/wI2L/jsondiff v0.3.0
 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 	google.golang.org/grpc v1.40.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.25
@@ -36,7 +37,6 @@ require (
 	k8s.io/client-go v0.22.3
 	k8s.io/cluster-bootstrap v0.22.3
 	k8s.io/component-base v0.22.3
-	k8s.io/component-helpers v0.22.3
 	k8s.io/controller-manager v0.22.3
 	k8s.io/klog/v2 v2.9.0
 	k8s.io/kube-controller-manager v0.22.3

diff --git a/go.sum b/go.sum
@@ -581,6 +581,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
+github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
@@ -590,6 +597,8 @@ github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb h1:MY3XXjEi7
 github.com/vishvananda/netlink v1.1.1-0.20200603190939-5a869a71f0cb/go.mod h1:FSQhuTO7eHT34mPzX+B04SUAjiqLxtXs1et0S6l9k4k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/wI2L/jsondiff v0.3.0 h1:iTzQ9u/d86GE9RsBzVHX88f2EA1vQUboHwLhSQFc1s4=
+github.com/wI2L/jsondiff v0.3.0/go.mod h1:y1IMzNNjlSsk3IUoJdRJO7VRBtzMvRgyo4Vu0LdHpTc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1146,8 +1155,6 @@ k8s.io/component-base v0.18.8/go.mod h1:00frPRDas29rx58pPCxNkhUfPbwajlyyvu8ruNgS
 k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA=
 k8s.io/component-base v0.22.3 h1:/+hryAW03u3FpJQww+GSMsArJNUbGjH66lrgxaRynLU=
 k8s.io/component-base v0.22.3/go.mod h1:kuybv1miLCMoOk3ebrqF93GbQHQx6W2287FC0YEQY6s=
-k8s.io/component-helpers v0.22.3 h1:08tn+T8HnjRTwDP2ErIBhHGvPcYJf5zWaWW83golHWc=
-k8s.io/component-helpers v0.22.3/go.mod h1:7OVySVH5elhHKuJKUOxZEfpT1Bm3ChmBQZHmuFfbGHk=
 k8s.io/controller-manager v0.22.3 h1:nBKG8MsgtUd/oFaZvE5zAYRIr45+Hn8QkHzq5+CtPOE=
 k8s.io/controller-manager v0.22.3/go.mod h1:4cvQGMvYf6IpTY08/NigEiI5UrN/cbtOe5e5WepYmcQ=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

diff --git a/pkg/controller/nodelifecycle/metrics.go b/pkg/controller/nodelifecycle/metrics.go