From 2bde1c0703dbeecf1a0ece0f5e0f23541e5d87ca Mon Sep 17 00:00:00 2001 From: nunu Date: Sun, 25 Dec 2022 01:09:16 +0800 Subject: [PATCH] make yurt-controller-manager take care of webhook configurations and certs --- charts/openyurt/templates/_helpers.tpl | 27 +++ .../job-patch/clusterrole.yaml | 32 ---- .../job-patch/clusterrolebinding.yaml | 24 --- .../job-patch/job-createSecret.yaml | 58 ------ .../job-patch/job-patchWebhook.yaml | 60 ------ .../admission-webhooks/job-patch/role.yaml | 25 --- .../job-patch/rolebinding.yaml | 25 --- .../job-patch/serviceaccount.yaml | 17 -- .../mutatingwebhookconfiguration.yaml | 28 --- .../validatingwebhookconfiguration.yaml | 27 --- .../admission-webhooks/webhookService.yaml | 22 --- .../templates/yurt-controller-manager.yaml | 81 ++++++-- charts/openyurt/values.yaml | 24 +-- pkg/controller/poolcoordinator/utils/file.go | 7 + pkg/webhook/certs.go | 166 +++++++++++++++++ pkg/webhook/poolcoordinator_webhook.go | 175 ++++++++++++++++-- pkg/webhook/webhook.go | 62 ++++--- 17 files changed, 466 insertions(+), 394 deletions(-) delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/clusterrole.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/clusterrolebinding.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/job-createSecret.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/job-patchWebhook.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/role.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/rolebinding.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/job-patch/serviceaccount.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/mutatingwebhookconfiguration.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/validatingwebhookconfiguration.yaml delete mode 100644 charts/openyurt/templates/admission-webhooks/webhookService.yaml create mode 100644 pkg/webhook/certs.go diff --git a/charts/openyurt/templates/_helpers.tpl b/charts/openyurt/templates/_helpers.tpl index 5b0982c9f5d..b17cd55fdff 100644 --- a/charts/openyurt/templates/_helpers.tpl +++ b/charts/openyurt/templates/_helpers.tpl @@ -1 +1,28 @@ {{/* vim: set filetype=mustache: */}} + +{{- define "yurt-controller-manager.fullname" -}} +yurt-controller-manager +{{- end -}} + +{{- define "yurt-controller-manager.name" -}} +yurt-controller-manager +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "yurt-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "yurt-controller-manager.name" . }} +app.kubernetes.io/instance: {{ printf "yurt-controller-manager-%s" .Release.Name }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "yurt-controller-manager.labels" -}} +{{ include "yurt-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} \ No newline at end of file diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/clusterrole.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/clusterrole.yaml deleted file mode 100644 index bfd81559427..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/clusterrole.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: pool-coordinator-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -rules: - - apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - - mutatingwebhookconfigurations - verbs: - - get - - update - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - update -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/clusterrolebinding.yaml deleted file mode 100644 index f88da90a895..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: pool-coordinator-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pool-coordinator-admission -subjects: - - kind: ServiceAccount - name: pool-coordinator-admission - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/job-createSecret.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/job-createSecret.yaml deleted file mode 100644 index 12d01390d78..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/job-createSecret.yaml +++ /dev/null @@ -1,58 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: pool-coordinator-admission-create - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission-create - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -spec: - template: - metadata: - name: pool-coordinator-admission-create - labels: - app: pool-coordinator-admission-create - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: create - image: {{ .Values.imageRegistry }}/{{ .Values.admissionWebhooks.patch.image.repository }}:{{ .Values.admissionWebhooks.patch.image.tag }} - imagePullPolicy: {{ .Values.admissionWebhooks.patch.image.pullPolicy }} - args: - - create - - --host=pool-coordinator-webhook,pool-coordinator-webhook.{{ .Release.Namespace }}.svc - - --namespace={{ .Release.Namespace }} - - --secret-name=pool-coordinator-admission - - --key-name=tls.key - - --cert-name=tls.crt - restartPolicy: OnFailure - serviceAccountName: pool-coordinator-admission - {{- with .Values.admissionWebhooks.patch.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.admissionWebhooks.patch.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - securityContext: - runAsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/job-patchWebhook.yaml deleted file mode 100644 index f0dda7fefa1..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: batch/v1 -kind: Job -metadata: - name: pool-coordinator-admission-patch - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission-patch - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -spec: - {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} - # Alpha feature since k8s 1.12 - ttlSecondsAfterFinished: 0 - {{- end }} - template: - metadata: - name: pool-coordinator-admission-patch - labels: - app: pool-coordinator-admission-patch - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm - spec: - containers: - - name: patch - image: {{ .Values.imageRegistry }}/{{ .Values.admissionWebhooks.patch.image.repository }}:{{ .Values.admissionWebhooks.patch.image.tag }} - imagePullPolicy: {{ .Values.admissionWebhooks.patch.image.pullPolicy }} - args: - - patch - - --webhook-name=pool-coordinator - - --namespace={{ .Release.Namespace }} - - --secret-name=pool-coordinator-admission - - --patch-failure-policy={{ .Values.admissionWebhooks.failurePolicy }} - - --patch-mutating=true - - --patch-validating=true - - --log-level=trace - restartPolicy: OnFailure - serviceAccountName: pool-coordinator-admission - {{- with .Values.admissionWebhooks.patch.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.admissionWebhooks.patch.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - securityContext: - runAsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/role.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/role.yaml deleted file mode 100644 index 47251bce43f..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/role.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: pool-coordinator-admission - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/rolebinding.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/rolebinding.yaml deleted file mode 100644 index da012a378f6..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/rolebinding.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: pool-coordinator-admission - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pool-coordinator-admission -subjects: - - kind: ServiceAccount - name: pool-coordinator-admission - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/job-patch/serviceaccount.yaml b/charts/openyurt/templates/admission-webhooks/job-patch/serviceaccount.yaml deleted file mode 100644 index 79526ec6c42..00000000000 --- a/charts/openyurt/templates/admission-webhooks/job-patch/serviceaccount.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if and .Values.admissionWebhooks.enabled .Values.admissionWebhooks.patch.enabled (not .Values.admissionWebhooks.certManager.enabled) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: pool-coordinator-admission - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: pool-coordinator-admission - helm.sh/chart: pool-coordinator-0.1.0 - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" - app.kubernetes.io/managed-by: Helm -{{- end }} diff --git a/charts/openyurt/templates/admission-webhooks/mutatingwebhookconfiguration.yaml b/charts/openyurt/templates/admission-webhooks/mutatingwebhookconfiguration.yaml deleted file mode 100644 index 66962215fe3..00000000000 --- a/charts/openyurt/templates/admission-webhooks/mutatingwebhookconfiguration.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: pool-coordinator -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - caBundle: Cg== - service: - name: pool-coordinator-webhook - namespace: {{ .Release.Namespace }} - path: /pool-coordinator-webhook-mutate - failurePolicy: Fail - name: mpoolcoordinator.kb.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - pods - sideEffects: None - diff --git a/charts/openyurt/templates/admission-webhooks/validatingwebhookconfiguration.yaml b/charts/openyurt/templates/admission-webhooks/validatingwebhookconfiguration.yaml deleted file mode 100644 index 25f5094741b..00000000000 --- a/charts/openyurt/templates/admission-webhooks/validatingwebhookconfiguration.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: pool-coordinator -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - caBundle: Cg== - service: - name: pool-coordinator-webhook - namespace: {{ .Release.Namespace }} - path: /pool-coordinator-webhook-validate - failurePolicy: Fail - name: vpoolcoordinator.kb.io - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - DELETE - resources: - - pods - sideEffects: None - diff --git a/charts/openyurt/templates/admission-webhooks/webhookService.yaml b/charts/openyurt/templates/admission-webhooks/webhookService.yaml deleted file mode 100644 index 5b1e85d4751..00000000000 --- a/charts/openyurt/templates/admission-webhooks/webhookService.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.admissionWebhooks.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: pool-coordinator-webhook - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - app.kubernetes.io/version: "1.0.0" -spec: - type: {{ .Values.admissionWebhooks.service.type }} - ports: - - port: 443 - targetPort: {{ .Values.admissionWebhooks.service.port }} - protocol: TCP - name: https - selector: - app.kubernetes.io/name: pool-coordinator - app.kubernetes.io/instance: pool-coordinator - -{{- end -}} diff --git a/charts/openyurt/templates/yurt-controller-manager.yaml b/charts/openyurt/templates/yurt-controller-manager.yaml index ad7ed0936ab..002beb0e734 100644 --- a/charts/openyurt/templates/yurt-controller-manager.yaml +++ b/charts/openyurt/templates/yurt-controller-manager.yaml @@ -142,6 +142,16 @@ rules: - get - list - watch + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -160,15 +170,18 @@ apiVersion: apps/v1 kind: Deployment metadata: name: yurt-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "yurt-controller-manager.labels" . | nindent 4 }} spec: replicas: 1 selector: matchLabels: - app: yurt-controller-manager + {{- include "yurt-controller-manager.selectorLabels" . | nindent 6 }} template: metadata: labels: - app: yurt-controller-manager + {{- include "yurt-controller-manager.selectorLabels" . | nindent 8 }} spec: serviceAccountName: yurt-controller-manager hostNetwork: true @@ -190,19 +203,53 @@ spec: image: "{{ .Values.yurtControllerManager.image.registry }}/{{ .Values.yurtControllerManager.image.repository }}:{{ .Values.yurtControllerManager.image.tag }}" imagePullPolicy: {{ .Values.yurtControllerManager.image.pullPolicy }} command: - - yurt-controller-manager - {{- if .Values.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} - {{- end }} + - yurt-controller-manager + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{ toYaml .Values.imagePullSecrets | nindent 10 }} + {{- end }} + ports: + - name: webhook-server + containerPort: {{ .Values.admissionWebhooks.service.port }} + protocol: TCP + - name: health + containerPort: 8000 + protocol: TCP env: - - name: WEBHOOK_CERT_DIR - value: {{ .Values.admissionWebhooks.certificate.mountPath }} - volumeMounts: - - mountPath: {{ .Values.admissionWebhooks.certificate.mountPath }} - name: cert - readOnly: true - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: pool-coordinator-admission + - name: WEBHOOK_CERT_DIR + value: {{ .Values.admissionWebhooks.certificate.mountPath }} + - name: WEBHOOK_SERVICE_PORT + value: {{ .Values.admissionWebhooks.service.port | quote }} + - name: WEBHOOK_SERVICE_NAME + value: {{ template "yurt-controller-manager.fullname" . }}-webhook + - name: WEBHOOK_NAMESPACE + value: {{ .Release.Namespace }} + - name: WEBHOOK_POD_VALIDATING_CONFIGURATION_NAME + value: {{ template "yurt-controller-manager.fullname" . }} + - name: WEBHOOK_POD_MUTATING_CONFIGURATION_NAME + value: {{ template "yurt-controller-manager.fullname" . }} + - name: WEBHOOK_POD_VALIDATING_NAME + value: {{ .Values.admissionWebhooks.names.validatingWebhookName }} + - name: WEBHOOK_POD_MUTATING_NAME + value: {{ .Values.admissionWebhooks.names.mutatingWebhookName }} + - name: WEBHOOK_POD_VALIDATING_PATH + value: {{ .Values.admissionWebhooks.names.webhookPodValidatingPath }} + - name: WEBHOOK_POD_MUTATING_PATH + value: {{ .Values.admissionWebhooks.names.webhookPodMutatingPath }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "yurt-controller-manager.name" . }}-webhook + namespace: {{ .Release.Namespace }} + labels: + {{- include "yurt-controller-manager.labels" . | nindent 4 }} +spec: + type: {{ .Values.admissionWebhooks.service.type }} + ports: + - port: 443 + targetPort: {{ .Values.admissionWebhooks.service.port }} + protocol: TCP + name: https + selector: + {{ include "yurt-controller-manager.selectorLabels" . | nindent 6 }} diff --git a/charts/openyurt/values.yaml b/charts/openyurt/values.yaml index 7eda7404386..6c3c89084d5 100644 --- a/charts/openyurt/values.yaml +++ b/charts/openyurt/values.yaml @@ -13,31 +13,17 @@ yurtControllerManager: port: 80 admissionWebhooks: - enabled: true service: type: ClusterIP port: 9443 failurePolicy: Fail certificate: mountPath: /tmp/k8s-webhook-server/serving-certs - patch: - enabled: true - image: - repository: docker.io/oamdev/kube-webhook-certgen - tag: v2.4.1 - pullPolicy: IfNotPresent - affinity: {} - tolerations: - [ - { - "key": "node-role.kubernetes.io/master", - "operator": "Exists", - "effect": "NoSchedule", - }, - ] - certManager: - enabled: false - revisionHistoryLimit: 3 + names: + validatingWebhookName: vpoolcoordinator.openyurt.io + mutatingWebhookName: mpoolcoordinator.openyurt.io + webhookPodValidatingPath: /pool-coordinator-webhook-validate + webhookPodMutatingPath: /pool-coordinator-webhook-mutate yurtTunnelAgent: replicaCount: 1 diff --git a/pkg/controller/poolcoordinator/utils/file.go b/pkg/controller/poolcoordinator/utils/file.go index 8d2982b8ca5..df456e7d907 100644 --- a/pkg/controller/poolcoordinator/utils/file.go +++ b/pkg/controller/poolcoordinator/utils/file.go @@ -21,6 +21,13 @@ import ( "os" ) +func GetEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { diff --git a/pkg/webhook/certs.go b/pkg/webhook/certs.go new file mode 100644 index 00000000000..dbccb3060f9 --- /dev/null +++ b/pkg/webhook/certs.go @@ -0,0 +1,166 @@ +/* +Copyright 2022 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math" + "math/big" + "net" + "time" + + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" +) + +const ( + CAKeyName = "ca-key.pem" + CACertName = "ca-cert.pem" + ServerKeyName = "key.pem" + ServerKeyName2 = "tls.key" + ServerCertName = "cert.pem" + ServerCertName2 = "tls.crt" +) + +type Certs struct { + // PEM encoded private key + Key []byte + // PEM encoded serving certificate + Cert []byte + // PEM encoded CA private key + CAKey []byte + // PEM encoded CA certificate + CACert []byte + // Resource version of the certs + ResourceVersion string +} + +const ( + rsaKeySize = 2048 +) + +// GenerateCerts generate a suite of self signed CA and server cert +func GenerateCerts(serviceNamespace, serviceName string) *Certs { + certs := &Certs{} + certs.generate(serviceNamespace, serviceName) + + return certs +} + +func (c *Certs) generate(serviceNamespace, serviceName string) error { + caKey, err := rsa.GenerateKey(rand.Reader, rsaKeySize) + if err != nil { + return fmt.Errorf("failed to create CA private key: %v", err) + } + caCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "yurt-webhooks-cert-ca"}, caKey) + if err != nil { + return fmt.Errorf("failed to create CA cert: %v", err) + } + + key, err := rsa.GenerateKey(rand.Reader, rsaKeySize) + if err != nil { + return fmt.Errorf("failed to create private key: %v", err) + } + + commonName := ServiceToCommonName(serviceNamespace, serviceName) + hostIP := net.ParseIP(commonName) + var altIPs []net.IP + if hostIP.To4() != nil { + altIPs = append(altIPs, hostIP.To4()) + } + dnsNames := []string{serviceName, fmt.Sprintf("%s.%s", serviceName, serviceNamespace), commonName} + cert, err := NewSignedCert( + cert.Config{ + CommonName: commonName, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + AltNames: cert.AltNames{IPs: altIPs, DNSNames: dnsNames}, + }, + key, caCert, caKey, + ) + if err != nil { + return fmt.Errorf("failed to create cert: %v", err) + } + + c.Key = EncodePrivateKeyPEM(key) + c.Cert = EncodeCertPEM(cert) + c.CAKey = EncodePrivateKeyPEM(caKey) + c.CACert = EncodeCertPEM(caCert) + + return nil +} + +func NewSignedCert(cfg cert.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 100).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +// EncodePrivateKeyPEM returns PEM-encoded private key data +func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { + block := pem.Block{ + Type: keyutil.RSAPrivateKeyBlockType, + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return pem.EncodeToMemory(&block) +} + +// EncodeCertPEM returns PEM-endcoded certificate data +func EncodeCertPEM(ct *x509.Certificate) []byte { + block := pem.Block{ + Type: cert.CertificateBlockType, + Bytes: ct.Raw, + } + return pem.EncodeToMemory(&block) +} + +// serviceToCommonName generates the CommonName for the certificate when using a k8s service. +func ServiceToCommonName(serviceNamespace, serviceName string) string { + return fmt.Sprintf("%s.%s.svc", serviceName, serviceNamespace) +} diff --git a/pkg/webhook/poolcoordinator_webhook.go b/pkg/webhook/poolcoordinator_webhook.go index f805e833746..7eca7deca5d 100644 --- a/pkg/webhook/poolcoordinator_webhook.go +++ b/pkg/webhook/poolcoordinator_webhook.go @@ -19,6 +19,7 @@ package webhook import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -26,7 +27,9 @@ import ( "github.com/wI2L/jsondiff" admissionv1 "k8s.io/api/admission/v1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" @@ -60,10 +63,7 @@ const ( // when ready nodes in a pool is below this value, we don't allow pod transition any more PoolReadyNodeNumberRatioThresholdDefault = 0.35 - ValidatePath string = "/pool-coordinator-webhook-validate" - MutatePath string = "/pool-coordinator-webhook-mutate" - HealthPath string = "/pool-coordinator-webhook-health" - MaxRetries = 30 + MaxRetries = 30 ) type PoolCoordinatorWebhook struct { @@ -78,6 +78,15 @@ type PoolCoordinatorWebhook struct { nodepoolMap *utils.NodepoolMap nodePoolUpdateQueue workqueue.RateLimitingInterface + + validatingConfigurationName string + mutatingConfigurationName string + validatingName string + mutatingName string + serviceName string + validatingPath string + mutatingPath string + namespace string } type validation struct { @@ -168,11 +177,13 @@ func (pa *PodAdmission) validateDel() (validation, error) { func (pa *PodAdmission) mutateAddToleration() ([]byte, error) { toadd := []corev1.Toleration{ {Key: "node.kubernetes.io/unreachable", - Operator: "Exists", - Effect: "NoExecute"}, + Operator: "Exists", + Effect: "NoExecute", + TolerationSeconds: nil}, {Key: "node.kubernetes.io/not-ready", - Operator: "Exists", - Effect: "NoExecute"}, + Operator: "Exists", + Effect: "NoExecute", + TolerationSeconds: nil}, } tols := pa.pod.Spec.Tolerations merged, changed := utils.MergeTolerations(tols, toadd) @@ -203,6 +214,10 @@ func (pa *PodAdmission) mutateReview() (*admissionv1.AdmissionReview, error) { return pa.reviewResponse(pa.request.UID, false, http.StatusBadRequest, ""), err } + if pa.node == nil { + return pa.reviewResponse(pa.request.UID, true, http.StatusAccepted, "node not assigned yet, nothing to do"), nil + } + if pa.request.Operation != admissionv1.Create && pa.request.Operation != admissionv1.Update { reason := fmt.Sprintf("Operation %v is accepted always", pa.request.Operation) return pa.reviewResponse(pa.request.UID, true, http.StatusAccepted, reason), nil @@ -214,6 +229,7 @@ func (pa *PodAdmission) mutateReview() (*admissionv1.AdmissionReview, error) { } // add tolerations if not yet + klog.Infof("add tolerations to pod %s\n", pa.pod.Name) val, err := pa.mutateAddToleration() if err != nil { return pa.reviewResponse(pa.request.UID, true, http.StatusAccepted, "could not merge tolerations"), err @@ -269,7 +285,6 @@ func (h *PoolCoordinatorWebhook) serveHealth(w http.ResponseWriter, r *http.Requ // ServeValidatePods validates an admission request and then writes an admission func (h *PoolCoordinatorWebhook) serveValidatePods(w http.ResponseWriter, r *http.Request) { klog.Info("uri", r.RequestURI) - klog.Info("received validation request") pa, err := h.NewPodAdmission(r) if err != nil { @@ -304,7 +319,6 @@ func (h *PoolCoordinatorWebhook) serveValidatePods(w http.ResponseWriter, r *htt // ServeMutatePods mutates an admission request and then writes an admission func (h *PoolCoordinatorWebhook) serveMutatePods(w http.ResponseWriter, r *http.Request) { klog.Info("uri", r.RequestURI) - klog.Info("received validation request") pa, err := h.NewPodAdmission(r) if err != nil { @@ -371,19 +385,27 @@ func (h *PoolCoordinatorWebhook) NewPodAdmission(r *http.Request) (*PodAdmission req := in.Request + if req.Kind.Kind != "Pod" { + return nil, fmt.Errorf("only pods are supported") + } + pod := &corev1.Pod{} - if err := json.Unmarshal(req.OldObject.Raw, pod); err != nil { - klog.Error(err) - return nil, err + if req.Operation == admissionv1.Delete || req.Operation == admissionv1.Update { + err = json.Unmarshal(req.OldObject.Raw, pod) + } else { + err = json.Unmarshal(req.Object.Raw, pod) } - nodeName := pod.Spec.NodeName - node, err := h.nodeLister.Get(nodeName) if err != nil { + klog.Error(err) return nil, err } + nodeName := pod.Spec.NodeSelector["kubernetes.io/hostname"] + klog.Infof("pod %s is on node: %s\n", pod.Name, nodeName) + node, _ := h.nodeLister.Get(nodeName) + pa := &PodAdmission{ request: req, pod: pod, @@ -412,6 +434,15 @@ func NewPoolcoordinatorWebhook(kc client.Interface, informerFactory informers.Sh h.nodepoolMap = utils.NewNodepoolMap() + h.serviceName = utils.GetEnv("WEBHOOK_SERVICE_NAME", "yurt-controller-manager-webhook") + h.namespace = utils.GetEnv("WEBHOOK_NAMESPACE", "kube-system") + h.validatingConfigurationName = utils.GetEnv("WEBHOOK_POD_VALIDATING_CONFIGURATION_NAME", "yurt-controller-manager") + h.mutatingConfigurationName = utils.GetEnv("WEBHOOK_POD_MUTATING_CONFIGURATION_NAME", "yurt-controller-manager") + h.validatingName = utils.GetEnv("WEBHOOK_POD_VALIDATING_NAME", "vpoolcoordinator.openyurt.io") + h.mutatingName = utils.GetEnv("WEBHOOK_POD_MUTATING_NAME", "mpoolcoordinator.openyurt.io") + h.validatingPath = utils.GetEnv("WEBHOOK_POD_VALIDATING_PATH", "/pool-coordinator-webhook-validate") + h.mutatingPath = utils.GetEnv("WEBHOOK_POD_MUTATING_PATH", "/pool-coordinator-webhook-mutate") + h.nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: h.onNodeCreate, UpdateFunc: h.onNodeUpdate, @@ -451,6 +482,7 @@ func (h *PoolCoordinatorWebhook) syncHandler(key string) error { if node == nil && err != nil { // the node has been deleted h.nodepoolMap.DelNode(name) + klog.Infof("node %s removed\n", name) return nil } @@ -461,11 +493,14 @@ func (h *PoolCoordinatorWebhook) syncHandler(key string) error { return nil } else { h.nodepoolMap.Del(opool, name) + klog.Infof("pool %s: node %s removed\n", opool, name) } } h.nodepoolMap.Add(pool, name) + klog.Infof("pool %s: node %s added\n", pool, name) } else { h.nodepoolMap.DelNode(name) + klog.Infof("node %s removed\n", name) } h.nodePoolUpdateQueue.Done(key) @@ -497,12 +532,113 @@ func (h *PoolCoordinatorWebhook) nodePoolWorker() { func (h *PoolCoordinatorWebhook) Handler() []Handler { return []Handler{ - {MutatePath, h.serveMutatePods}, - {ValidatePath, h.serveMutatePods}, + {h.mutatingPath, h.serveMutatePods}, + {h.validatingPath, h.serveValidatePods}, + } +} + +func (h *PoolCoordinatorWebhook) ensureValidatingConfiguration(certs *Certs) { + fail := admissionregistrationv1.Fail + sideEffects := admissionregistrationv1.SideEffectClassNone + config := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: h.validatingConfigurationName, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: h.validatingName, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + CABundle: certs.CACert, + Service: &admissionregistrationv1.ServiceReference{ + Name: h.serviceName, + Namespace: h.namespace, + Path: &h.validatingPath, + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + {Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Delete}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + FailurePolicy: &fail, + SideEffects: &sideEffects, + AdmissionReviewVersions: []string{"v1"}, + }}, + } + + if _, err := h.client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get( + context.TODO(), h.validatingConfigurationName, metav1.GetOptions{}); err != nil { + if errors.IsNotFound(err) { + klog.Infof("validatewebhookconfiguratiion %s not found, create it.", h.validatingConfigurationName) + if _, err = h.client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create( + context.TODO(), config, metav1.CreateOptions{}); err != nil { + klog.Fatal(err) + } + } + } else { + klog.Infof("validatingwebhookconfiguratiion %s already exists, update it.", h.validatingConfigurationName) + if _, err = h.client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update( + context.TODO(), config, metav1.UpdateOptions{}); err != nil { + klog.Fatal(err) + } } } -func (h *PoolCoordinatorWebhook) Init(stopCH <-chan struct{}) { +func (h *PoolCoordinatorWebhook) ensureMutatingConfiguration(certs *Certs) { + fail := admissionregistrationv1.Fail + sideEffects := admissionregistrationv1.SideEffectClassNone + config := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: h.mutatingConfigurationName, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{{ + Name: h.mutatingName, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + CABundle: certs.CACert, + Service: &admissionregistrationv1.ServiceReference{ + Name: h.serviceName, + Namespace: h.namespace, + Path: &h.mutatingPath, + }, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + {Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + FailurePolicy: &fail, + SideEffects: &sideEffects, + AdmissionReviewVersions: []string{"v1"}, + }}, + } + + if _, err := h.client.AdmissionregistrationV1().MutatingWebhookConfigurations().Get( + context.TODO(), h.mutatingConfigurationName, metav1.GetOptions{}); err != nil { + if errors.IsNotFound(err) { + klog.Infof("validatewebhookconfiguratiion %s not found, create it.", h.mutatingConfigurationName) + if _, err = h.client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create( + context.TODO(), config, metav1.CreateOptions{}); err != nil { + klog.Fatal(err) + } + } + } else { + klog.Infof("validatingwebhookconfiguratiion %s already exists, update it.", h.mutatingConfigurationName) + if _, err = h.client.AdmissionregistrationV1().MutatingWebhookConfigurations().Update( + context.TODO(), config, metav1.UpdateOptions{}); err != nil { + klog.Fatal(err) + } + } +} + +func (h *PoolCoordinatorWebhook) Init(certs *Certs, stopCH <-chan struct{}) { if !cache.WaitForCacheSync(stopCH, h.nodeSynced, h.leaseSynced) { klog.Error("sync poolcoordinator webhook timeout") } @@ -521,4 +657,7 @@ func (h *PoolCoordinatorWebhook) Init(stopCH <-chan struct{}) { defer h.nodePoolUpdateQueue.ShutDown() <-stopCH }() + + h.ensureValidatingConfiguration(certs) + h.ensureMutatingConfiguration(certs) } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 58a11836f58..46a3bc7dafb 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -18,8 +18,8 @@ limitations under the License. package webhook import ( + "fmt" "net/http" - "os" "time" "k8s.io/client-go/informers" @@ -29,11 +29,6 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/poolcoordinator/utils" ) -const ( - CertDirEnvName = "WEBHOOK_CERT_DIR" - DefaultCertDir = "/tmp/k8s-webhook-server/serving-certs" -) - type Handler struct { Path string HttpHandler func(http.ResponseWriter, *http.Request) @@ -41,7 +36,7 @@ type Handler struct { type Webhook interface { Handler() []Handler - Init(<-chan struct{}) + Init(*Certs, <-chan struct{}) } type WebhookManager struct { @@ -50,13 +45,26 @@ type WebhookManager struct { webhooks []Webhook } +func webhookCertDir() string { + return utils.GetEnv("WEBHOOK_CERT_DIR", "/tmp/k8s-webhook-server/serving-certs") +} + +func webhookNamespace() string { + return utils.GetEnv("WEBHOOK_NAMESPACE", "kube-system") +} + +func webhookServiceName() string { + return utils.GetEnv("WEBHOOK_SERVICE_NAME", "yurt-controller-manager-webhook") +} + +func webhookServicePort() string { + return utils.GetEnv("WEBHOOK_SERVICE_PORT", "9443") +} + func NewWebhookManager(kc client.Interface, informerFactory informers.SharedInformerFactory) *WebhookManager { m := &WebhookManager{} - m.certdir = os.Getenv(CertDirEnvName) - if m.certdir == "" { - m.certdir = DefaultCertDir - } + m.certdir = webhookCertDir() h := NewPoolcoordinatorWebhook(kc, informerFactory) m.addWebhook(h) @@ -69,22 +77,25 @@ func (m *WebhookManager) addWebhook(webhook Webhook) { } func (m *WebhookManager) Run(stopCH <-chan struct{}) { - for _, h := range m.webhooks { - h.Init(stopCH) - for _, hh := range h.Handler() { - http.HandleFunc(hh.Path, hh.HttpHandler) - } - } - err := utils.EnsureDir(m.certdir) if err != nil { klog.Error(err) } - cert := m.certdir + "/tls.crt" + crt := m.certdir + "/tls.crt" key := m.certdir + "/tls.key" + certs := GenerateCerts(webhookNamespace(), webhookServiceName()) + err = utils.WriteFile(crt, certs.Cert) + if err != nil { + klog.Error(err) + } + err = utils.WriteFile(key, certs.Key) + if err != nil { + klog.Error(err) + } + for { - if utils.FileExists(cert) && utils.FileExists(key) { + if utils.FileExists(crt) && utils.FileExists(key) { klog.Info("tls key and cert ok.") break } else { @@ -93,6 +104,13 @@ func (m *WebhookManager) Run(stopCH <-chan struct{}) { } } - klog.Info("Listening on port 443...") - klog.Fatal(http.ListenAndServeTLS(":443", cert, key, nil)) + for _, h := range m.webhooks { + h.Init(certs, stopCH) + for _, hh := range h.Handler() { + http.HandleFunc(hh.Path, hh.HttpHandler) + } + } + + klog.Infof("Listening on port %s ...", webhookServicePort()) + klog.Fatal(http.ListenAndServeTLS(fmt.Sprintf(":%s", webhookServicePort()), crt, key, nil)) }