From eb3eab70bb8e2fedd9444d6131d328066d85bc37 Mon Sep 17 00:00:00 2001 From: star Date: Mon, 11 Oct 2021 20:12:08 +0800 Subject: [PATCH] Refactor: modify certificate codes (#514) --- .../app/controllermanager.go | 2 +- cmd/yurt-controller-manager/app/core.go | 4 +- cmd/yurt-tunnel-server/app/start.go | 21 +- pkg/controller/certificates/csrapprover.go | 64 ++--- .../certificates/csrapprover_test.go | 230 +++++++++++++++++ pkg/projectinfo/projectinfo.go | 19 +- pkg/yurttunnel/pki/certmanager/csrapprover.go | 234 ------------------ 7 files changed, 285 insertions(+), 289 deletions(-) create mode 100644 pkg/controller/certificates/csrapprover_test.go delete mode 100644 pkg/yurttunnel/pki/certmanager/csrapprover.go diff --git a/cmd/yurt-controller-manager/app/controllermanager.go b/cmd/yurt-controller-manager/app/controllermanager.go index ff2513b7f1f..d7d40c0f8a8 100644 --- a/cmd/yurt-controller-manager/app/controllermanager.go +++ b/cmd/yurt-controller-manager/app/controllermanager.go @@ -287,7 +287,7 @@ var ControllersDisabledByDefault = sets.NewString() func NewControllerInitializers() map[string]InitFunc { controllers := map[string]InitFunc{} controllers["nodelifecycle"] = startNodeLifecycleController - controllers["yurthubcsrapprover"] = startYurtHubCSRApproverController + controllers["yurtcsrapprover"] = startYurtCSRApproverController return controllers } diff --git a/cmd/yurt-controller-manager/app/core.go b/cmd/yurt-controller-manager/app/core.go index d76087d1558..4986e970502 100644 --- a/cmd/yurt-controller-manager/app/core.go +++ b/cmd/yurt-controller-manager/app/core.go @@ -55,10 +55,10 @@ func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, er return nil, true, nil } -func startYurtHubCSRApproverController(ctx ControllerContext) (http.Handler, bool, error) { +func startYurtCSRApproverController(ctx ControllerContext) (http.Handler, bool, error) { clientSet := ctx.ClientBuilder.ClientOrDie("csr-controller") go certificates.NewCSRApprover(clientSet, ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests()). - Run(certificates.YurtHubCSRApproverThreadiness, ctx.Stop) + Run(certificates.YurtCSRApproverThreadiness, ctx.Stop) return nil, true, nil } diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index a28e79a80a6..132436e6d17 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -24,7 +24,6 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config" "github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" "github.com/openyurtio/openyurt/pkg/yurttunnel/dns" "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/initializer" "github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/wraphandler" @@ -33,8 +32,8 @@ import ( "github.com/openyurtio/openyurt/pkg/yurttunnel/pki/certmanager" "github.com/openyurtio/openyurt/pkg/yurttunnel/server" "github.com/openyurtio/openyurt/pkg/yurttunnel/util" - "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) @@ -109,16 +108,8 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { return err } serverCertMgr.Start() - go certmanager.NewCSRApprover(cfg.Client, cfg.SharedInformerFactory.Certificates().V1beta1().CertificateSigningRequests()). - Run(constants.YurttunnelCSRApproverThreadiness, stopCh) - - // 3. generate the TLS configuration based on the latest certificate - tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert) - if err != nil { - return err - } - // 4. create handler wrappers + // 3. create handler wrappers mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory) wrappers, err := wraphandler.InitHandlerWrappers(mInitializer) if err != nil { @@ -128,7 +119,7 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { // after all of informers are configured completed, start the shared index informer cfg.SharedInformerFactory.Start(stopCh) - // 5. waiting for the certificate is generated + // 4. waiting for the certificate is generated _ = wait.PollUntil(5*time.Second, func() (bool, error) { // keep polling until the certificate is signed if serverCertMgr.Current() != nil { @@ -138,6 +129,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { return false, nil }, stopCh) + // 5. generate the TLS configuration based on the latest certificate + tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert) + if err != nil { + return err + } + // 6. start the server ts := server.NewTunnelServer( cfg.EgressSelectorEnabled, diff --git a/pkg/controller/certificates/csrapprover.go b/pkg/controller/certificates/csrapprover.go index 933bd27a8a4..3aba335d131 100644 --- a/pkg/controller/certificates/csrapprover.go +++ b/pkg/controller/certificates/csrapprover.go @@ -37,23 +37,25 @@ import ( "k8s.io/klog" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server" ) const ( - YurtHubCSRApproverThreadiness = 2 + // yurthub PKI related constants + YurthubCSROrg = "openyurt:yurthub" + // yurttunnel PKI related constants + YurttunnelCSROrg = "openyurt:yurttunnel" + YurtCSRApproverThreadiness = 2 ) -// YurtHubCSRApprover is the controller that auto approve all -// yurthub related CSR -type YurtHubCSRApprover struct { +// YurtCSRApprover is the controller that auto approve all openyurt related CSR +type YurtCSRApprover struct { csrInformer certv1beta1.CertificateSigningRequestInformer csrClient typev1beta1.CertificateSigningRequestInterface workqueue workqueue.RateLimitingInterface } -// Run starts the YurtHubCSRApprover -func (yca *YurtHubCSRApprover) Run(threadiness int, stopCh <-chan struct{}) { +// Run starts the YurtCSRApprover +func (yca *YurtCSRApprover) Run(threadiness int, stopCh <-chan struct{}) { defer runtime.HandleCrash() defer yca.workqueue.ShutDown() klog.Info("starting the crsapprover") @@ -69,12 +71,12 @@ func (yca *YurtHubCSRApprover) Run(threadiness int, stopCh <-chan struct{}) { klog.Info("stoping the csrapprover") } -func (yca *YurtHubCSRApprover) runWorker() { +func (yca *YurtCSRApprover) runWorker() { for yca.processNextItem() { } } -func (yca *YurtHubCSRApprover) processNextItem() bool { +func (yca *YurtCSRApprover) processNextItem() bool { key, quit := yca.workqueue.Get() if quit { return false @@ -97,7 +99,7 @@ func (yca *YurtHubCSRApprover) processNextItem() bool { return true } - if err := approveYurtHubCSR(csr, yca.csrClient); err != nil { + if err := approveCSR(csr, yca.csrClient); err != nil { runtime.HandleError(err) enqueueObj(yca.workqueue, csr) return true @@ -119,8 +121,8 @@ func enqueueObj(wq workqueue.RateLimitingInterface, obj interface{}) { return } - if !isYurtHubCSR(csr) { - klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetHubName()) + if !isYurtCSR(csr) { + klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetProjectPrefix()) return } @@ -133,10 +135,10 @@ func enqueueObj(wq workqueue.RateLimitingInterface, obj interface{}) { klog.V(4).Infof("approved or denied csr, ignore it: %s", key) } -// NewCSRApprover creates a new YurtHubCSRApprover +// NewCSRApprover creates a new YurtCSRApprover func NewCSRApprover( clientset kubernetes.Interface, - csrInformer certinformer.CertificateSigningRequestInformer) *YurtHubCSRApprover { + csrInformer certinformer.CertificateSigningRequestInformer) *YurtCSRApprover { wq := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) csrInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -147,25 +149,26 @@ func NewCSRApprover( enqueueObj(wq, new) }, }) - return &YurtHubCSRApprover{ + return &YurtCSRApprover{ csrInformer: csrInformer, csrClient: clientset.CertificatesV1beta1().CertificateSigningRequests(), workqueue: wq, } } -// approveYurtHubCSR checks the csr status, if it is neither approved nor +// approveCSR checks the csr status, if it is neither approved nor // denied, it will try to approve the csr. -func approveYurtHubCSR( +func approveCSR( obj interface{}, csrClient typev1beta1.CertificateSigningRequestInterface) error { csr, ok := obj.(*certificates.CertificateSigningRequest) if !ok { + klog.Infof("object is not csr: %v", obj) return nil } - if !isYurtHubCSR(csr) { - klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetHubName()) + if !isYurtCSR(csr) { + klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetProjectPrefix()) return nil } @@ -180,26 +183,26 @@ func approveYurtHubCSR( return nil } - // approve the yurthub related csr + // approve the openyurt related csr csr.Status.Conditions = append(csr.Status.Conditions, certificates.CertificateSigningRequestCondition{ Type: certificates.CertificateApproved, Reason: "AutoApproved", - Message: fmt.Sprintf("self-approving %s csr", projectinfo.GetHubName()), + Message: fmt.Sprintf("self-approving %s csr", projectinfo.GetProjectPrefix()), }) result, err := csrClient.UpdateApproval(context.Background(), csr, metav1.UpdateOptions{}) if err != nil { - klog.Errorf("failed to approve %s csr(%s), %v", projectinfo.GetHubName(), csr.GetName(), err) + klog.Errorf("failed to approve %s csr(%s), %v", projectinfo.GetProjectPrefix(), csr.GetName(), err) return err } - klog.Infof("successfully approve %s csr(%s)", projectinfo.GetHubName(), result.Name) + klog.Infof("successfully approve %s csr(%s)", projectinfo.GetProjectPrefix(), result.Name) return nil } -// isYurtHubCSR checks if given csr is a yurthub related csr, i.e., +// isYurtCSR checks if given csr is a openyurt related csr, i.e., // the organizations' list contains "openyurt:yurthub" -func isYurtHubCSR(csr *certificates.CertificateSigningRequest) bool { +func isYurtCSR(csr *certificates.CertificateSigningRequest) bool { pemBytes := csr.Spec.Request block, _ := pem.Decode(pemBytes) if block == nil || block.Type != "CERTIFICATE REQUEST" { @@ -209,15 +212,12 @@ func isYurtHubCSR(csr *certificates.CertificateSigningRequest) bool { if err != nil { return false } - for i, org := range x509cr.Subject.Organization { - if org == server.YurtHubCSROrg { - break - } - if i == len(x509cr.Subject.Organization)-1 { - return false + for _, org := range x509cr.Subject.Organization { + if org == YurttunnelCSROrg || org == YurthubCSROrg { + return true } } - return true + return false } // checkCertApprovalCondition checks if the given csr's status is diff --git a/pkg/controller/certificates/csrapprover_test.go b/pkg/controller/certificates/csrapprover_test.go new file mode 100644 index 00000000000..18e26593595 --- /dev/null +++ b/pkg/controller/certificates/csrapprover_test.go @@ -0,0 +1,230 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certificates + +import ( + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "log" + "testing" + + certificates "k8s.io/api/certificates/v1beta1" +) + +var rsaPrivateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA3xVbRYwmRaFAVUHRm/ynFbOe6pDNEsDIoEE2+7LDrlRndp1w +hzrOd9DPWBcEJIO6ga1U9TdCP1HnOWQLaoM4c4Tngb26ieZYNW2PKhij3wdoN5eO +t7jKupAD1eEChDjsZSN2/OJVLi9/82vAxjmfCzz9icRGlUd2E4Ixtd+EUxz4gCjQ +elNyjEPO28/6TfL3o18jX4UKonk+CKEIotrf1hph0I2/Feb+DeUQIRjvhwzoaauk +2epAUAeunMpatLkwQp6BfDTu/+MkJgcgyHv2qlb+2zYSvvzbVm3lNIa4Mkoe8Bqe +ecLxrp07uxp13SVtJE9EeyVdAIwNg0H7DJ6QDQIDAQABAoIBAA2scHC922avMJNJ +OoDWJqOk49u6zmcU2/c+qBEbbvUThVf25HvVdexQJzVeC8n1LQxfxHJXVb8t1P9m +i3CW5HHoNox0RafIL6XutjS9V+YGvTOTHZNTR1HSG/oTFaVnG85DMzri4Je5H52b +ADDmPUJiFaRJHI5v1+PwOf3M2n6BjcRh9rIwDX/1eSyCWwKc10ZcXWcGTwyXtDz5 +lwIyKrUvmuQvjS6Wq84J6dTDQfk2LrXHk92UYaUyrpdQ5lvuOAAfvO8XoOmJvFf+ +h1RLnFdk5DB7aWH+DbQvfVEuoTgbhWgRHku/KpdITjTE0ZHp945hINnN/071W5aq +dHo5kx0CgYEA+HE05pV9eAlblq534H586wsCJHKB70Fi3yN2eQdJelwIeBkRwulW +hfLkaXQM/I8BsGgVviBfREq7Zzaz3v5UYirjJOe/X11b9mn7HGcVlNR0ilOKmVrH +SAWr/ZkjIiza25SRYJ/I1y0HE3GMGOdiwj52E4mEMXaExPW762p/AKMCgYEA5d6p +yqguhKzoBRFFA0CXARxu2uTfipfIYIJFAnYVg1fkJocV7mZP8z+qLc1qWUm8Enje +QagAbEZH5SDpEKiHGIGqmODl+qAg9vHXrMOcQabmwGhXK0wn1E3QRoVHQ8N90weB +9A/Mc7LKxDpSwTkWgVMJWxK29U75CXWfWjeAR48CgYAncqI5sqbXdnTqeg1iwfLH +x1mxu9TRzooKcDERioyqNw7JMwHU9wPcBPMro1ekinh0MDKzm6REzbDv9Ime8Lcp +VzH13C5Q0BwYBj/vBJcyqIFQrW8mZnmZ//yNKdGgTYr6rp5ev0A+mlGzTqY2Fhdi +TFSnSYCJ8g2m0HXkLWa5DQKBgQCZxJlQN7Dmj8OloCfKRSq+U4bUZsYir+YaqQoA +230In4K/Qx4om8hfr/bnLMI3eFuW/8Otp/SgeWMeoyVFP3cfrZ2xJsCxJuzmRGFB +8JhWUo+JpkKpdAgwvNzWT9GcQumogR0tZmQeATwih+FT4Bxt5l4bzikVb/6nlUdD +0ly9gQKBgBMBjd83IEJDBrUtWaxtPlt8HDFgTlqgZJxIckW3bnF+7iPTlO7hLpOD +dbALa7x9+2ydqd9lpyd8txi57gYMHuVi1KaBvMDbKAo0SXLNV1Kv73HNO3k/o2+w +k6IJMsIcAOOuF9N1A6awc8mEKiQ53slCbdosjes2Zurzv6gJGLQ+ +-----END RSA PRIVATE KEY-----` + +func TestIsYurtCSR(t *testing.T) { + tests := []struct { + desc string + csr *certificates.CertificateSigningRequest + exp bool + }{ + { + desc: "is not certificate request", + csr: &certificates.CertificateSigningRequest{ + Spec: certificates.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory( + &pem.Block{ + Type: "PUBLIC KEY", + }), + }, + }, + exp: false, + }, + { + desc: "can not parse certificate request", + csr: &certificates.CertificateSigningRequest{ + Spec: certificates.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: []byte(`MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlRuRnThUjU8/prwYxbty`), + }), + }, + }, + exp: false, + }, + { + desc: "is not a openyurt related certificate request", + csr: &certificates.CertificateSigningRequest{ + Spec: certificates.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: newCSR([]string{"not_openyurt_organization"}), + }), + }, + }, + exp: false, + }, + { + desc: "is yurttunnel related certificate request", + csr: &certificates.CertificateSigningRequest{ + Spec: certificates.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: newCSR([]string{YurttunnelCSROrg}), + }), + }, + }, + exp: true, + }, + { + desc: "is yurthub related certificate request", + csr: &certificates.CertificateSigningRequest{ + Spec: certificates.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: newCSR([]string{YurthubCSROrg}), + }), + }, + }, + exp: true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + act := isYurtCSR(tt.csr) + if act != tt.exp { + t.Errorf("the value we want is %v, but the actual value is %v", tt.exp, act) + } + }) + } +} + +func TestCheckCertApprovalCondition(t *testing.T) { + tests := []struct { + desc string + status *certificates.CertificateSigningRequestStatus + exp struct { + approved bool + denied bool + } + }{ + { + desc: "approved", + status: &certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + { + Type: certificates.CertificateApproved, + }, + }, + }, + exp: struct { + approved bool + denied bool + }{approved: true, denied: false}, + }, + { + desc: "denied", + status: &certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + { + Type: certificates.CertificateDenied, + }, + }, + }, + exp: struct { + approved bool + denied bool + }{approved: false, denied: true}, + }, + { + desc: "approved and denied", + status: &certificates.CertificateSigningRequestStatus{ + Conditions: []certificates.CertificateSigningRequestCondition{ + { + Type: certificates.CertificateApproved, + }, + { + Type: certificates.CertificateDenied, + }, + }, + }, + exp: struct { + approved bool + denied bool + }{approved: true, denied: true}, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + actApproved, actDenied := checkCertApprovalCondition(tt.status) + act := struct { + approved bool + denied bool + }{approved: actApproved, denied: actDenied} + if act != tt.exp { + t.Errorf("the value we want is %+v, but the actual value is %+v", tt.exp, act) + } + }) + } +} + +func newCSR(organizations []string) []byte { + block, _ := pem.Decode([]byte(rsaPrivateKey)) + rsaPriv, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + log.Fatalf("Failed to parse private key: %s", err) + } + + req := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: organizations, + }, + DNSNames: []string{ + "openyurt.io", + }, + } + csr, err := x509.CreateCertificateRequest(rand.Reader, req, rsaPriv) + if err != nil { + log.Fatalf("unable to create CSR: %s", err) + } + return csr +} diff --git a/pkg/projectinfo/projectinfo.go b/pkg/projectinfo/projectinfo.go index acfe1fb75a8..a46bcb5d9d0 100644 --- a/pkg/projectinfo/projectinfo.go +++ b/pkg/projectinfo/projectinfo.go @@ -45,46 +45,49 @@ func ShortServerVersion() string { return GetServerName() + "/" + gitVersion + "-" + commit } +// The project prefix is: yurt func GetProjectPrefix() string { return projectPrefix } +// Server name: yurttunnel-server func GetServerName() string { return projectPrefix + "tunnel-server" } +// Agent name: yurttunnel-agent func GetAgentName() string { return projectPrefix + "tunnel-agent" } -// GetEdgeWorkerLabelKey returns the edge-worker label, which is used to -// identify if a node is a edge node ("true") or a cloud node ("false") +// GetEdgeWorkerLabelKey returns the edge-worker label ("openyurt.io/is-edge-worker"), +// which is used to identify if a node is a edge node ("true") or a cloud node ("false") func GetEdgeWorkerLabelKey() string { return labelPrefix + "/is-edge-worker" } -// GetHubName returns name of yurthub agent +// GetHubName returns name of yurthub agent: yurthub func GetHubName() string { return projectPrefix + "hub" } -// GetEdgeEnableTunnelLabelKey returns the tunnel agent label, which is used -// to identify if tunnel agent is running on the node or not. +// GetEdgeEnableTunnelLabelKey returns the tunnel agent label ("openyurt.io/edge-enable-reverseTunnel-client"), +// which is used to identify if tunnel agent is running on the node or not. func GetEdgeEnableTunnelLabelKey() string { return labelPrefix + "/edge-enable-reverseTunnel-client" } -// GetTunnelName returns name of tunnel +// GetTunnelName returns name of tunnel: yurttunnel func GetTunnelName() string { return projectPrefix + "tunnel" } -// GetYurtControllerManagerName returns name of openyurt controller-manager +// GetYurtControllerManagerName returns name of openyurt controller-manager: yurtcontroller-manager func GetYurtControllerManagerName() string { return projectPrefix + "controller-manager" } -// GetYurtAppManagerName returns name of tunnel +// GetYurtAppManagerName returns name of tunnel: yurtapp-manager func GetYurtAppManagerName() string { return projectPrefix + "app-manager" } diff --git a/pkg/yurttunnel/pki/certmanager/csrapprover.go b/pkg/yurttunnel/pki/certmanager/csrapprover.go deleted file mode 100644 index e8d98f2bc56..00000000000 --- a/pkg/yurttunnel/pki/certmanager/csrapprover.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certmanager - -import ( - "context" - "crypto/x509" - "encoding/pem" - "fmt" - "time" - - certificates "k8s.io/api/certificates/v1beta1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - certinformer "k8s.io/client-go/informers/certificates/v1beta1" - certv1beta1 "k8s.io/client-go/informers/certificates/v1beta1" - "k8s.io/client-go/kubernetes" - typev1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" -) - -// YurttunnelCSRApprover is the controller that auto approve all -// yurttunnel related CSR -type YurttunnelCSRApprover struct { - csrInformer certv1beta1.CertificateSigningRequestInformer - csrClient typev1beta1.CertificateSigningRequestInterface - workqueue workqueue.RateLimitingInterface -} - -// Run starts the YurttunnelCSRApprover -func (yca *YurttunnelCSRApprover) Run(threadiness int, stopCh <-chan struct{}) { - defer runtime.HandleCrash() - defer yca.workqueue.ShutDown() - klog.Info("starting the crsapprover") - if !cache.WaitForCacheSync(stopCh, - yca.csrInformer.Informer().HasSynced) { - klog.Error("sync csr timeout") - return - } - for i := 0; i < threadiness; i++ { - go wait.Until(yca.runWorker, time.Second, stopCh) - } - <-stopCh - klog.Info("stoping the csrapprover") -} - -func (yca *YurttunnelCSRApprover) runWorker() { - for yca.processNextItem() { - } -} - -func (yca *YurttunnelCSRApprover) processNextItem() bool { - key, quit := yca.workqueue.Get() - if quit { - return false - } - csrName, ok := key.(string) - if !ok { - yca.workqueue.Forget(key) - runtime.HandleError( - fmt.Errorf("expected string in workqueue but got %#v", key)) - return true - } - defer yca.workqueue.Done(key) - - csr, err := yca.csrInformer.Lister().Get(csrName) - if err != nil { - runtime.HandleError(err) - if !apierrors.IsNotFound(err) { - yca.workqueue.AddRateLimited(key) - } - return true - } - - if err := approveYurttunnelCSR(csr, yca.csrClient); err != nil { - runtime.HandleError(err) - enqueueObj(yca.workqueue, csr) - return true - } - - return true -} - -func enqueueObj(wq workqueue.RateLimitingInterface, obj interface{}) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - runtime.HandleError(err) - return - } - - csr, ok := obj.(*certificates.CertificateSigningRequest) - if !ok { - klog.Errorf("%s is not a csr", key) - return - } - - if !isYurttunelCSR(csr) { - klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetTunnelName()) - return - } - - approved, denied := checkCertApprovalCondition(&csr.Status) - if !approved && !denied { - klog.Infof("non-approved and non-denied csr, enqueue: %s", key) - wq.AddRateLimited(key) - } - - klog.V(4).Infof("approved or denied csr, ignore it: %s", key) -} - -// NewCSRApprover creates a new YurttunnelCSRApprover -func NewCSRApprover( - clientset kubernetes.Interface, - csrInformer certinformer.CertificateSigningRequestInformer) *YurttunnelCSRApprover { - - wq := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - csrInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - enqueueObj(wq, obj) - }, - UpdateFunc: func(old, new interface{}) { - enqueueObj(wq, new) - }, - }) - return &YurttunnelCSRApprover{ - csrInformer: csrInformer, - csrClient: clientset.CertificatesV1beta1().CertificateSigningRequests(), - workqueue: wq, - } -} - -// approveYurttunnelCSR checks the csr status, if it is neither approved nor -// denied, it will try to approve the csr. -func approveYurttunnelCSR( - obj interface{}, - csrClient typev1beta1.CertificateSigningRequestInterface) error { - csr, ok := obj.(*certificates.CertificateSigningRequest) - if !ok { - klog.Infof("object is not csr: %v", obj) - return nil - } - - if !isYurttunelCSR(csr) { - klog.Infof("csr(%s) is not %s csr", csr.GetName(), projectinfo.GetTunnelName()) - return nil - } - - approved, denied := checkCertApprovalCondition(&csr.Status) - if approved { - klog.Infof("csr(%s) is approved", csr.GetName()) - return nil - } - - if denied { - klog.Infof("csr(%s) is denied", csr.GetName()) - return nil - } - - // approve the yurttunnel related csr - csr.Status.Conditions = append(csr.Status.Conditions, - certificates.CertificateSigningRequestCondition{ - Type: certificates.CertificateApproved, - Reason: "AutoApproved", - Message: fmt.Sprintf("self-approving %s csr", projectinfo.GetTunnelName()), - }) - - result, err := csrClient.UpdateApproval(context.Background(), csr, metav1.UpdateOptions{}) - if err != nil { - klog.Errorf("failed to approve %s csr(%s), %v", projectinfo.GetTunnelName(), csr.GetName(), err) - return err - } - klog.Infof("successfully approve %s csr(%s)", projectinfo.GetTunnelName(), result.Name) - return nil -} - -// isYurttunelCSR checks if given csr is a yurttunnel related csr, i.e., -// the organizations' list contains "openyurt:yurttunnel" -func isYurttunelCSR(csr *certificates.CertificateSigningRequest) bool { - pemBytes := csr.Spec.Request - block, _ := pem.Decode(pemBytes) - if block == nil || block.Type != "CERTIFICATE REQUEST" { - return false - } - x509cr, err := x509.ParseCertificateRequest(block.Bytes) - if err != nil { - return false - } - for i, org := range x509cr.Subject.Organization { - if org == constants.YurttunnelCSROrg { - break - } - if i == len(x509cr.Subject.Organization)-1 { - return false - } - } - return true -} - -// checkCertApprovalCondition checks if the given csr's status is -// approved or denied -func checkCertApprovalCondition( - status *certificates.CertificateSigningRequestStatus) ( - approved bool, denied bool) { - for _, c := range status.Conditions { - if c.Type == certificates.CertificateApproved { - approved = true - } - if c.Type == certificates.CertificateDenied { - denied = true - } - } - return -}