From 5ba736c8cce97e036ed17848f8b0bcfe2a2355b8 Mon Sep 17 00:00:00 2001 From: rambohe Date: Thu, 12 Jan 2023 15:28:25 +0800 Subject: [PATCH] Improve certificate manager (#1133) * improve certificate manager for yurthub * fix hub root dir comment --- cmd/yurt-tunnel-server/app/start.go | 4 +- cmd/yurthub/app/config/config.go | 7 +- cmd/yurthub/app/options/options.go | 15 +- cmd/yurthub/app/start.go | 21 +- pkg/controller/certificates/csrapprover.go | 4 +- .../certificates/csrapprover_test.go | 6 +- pkg/node-servant/components/yurthub.go | 7 +- pkg/util/certmanager/factory/factory.go | 33 +- pkg/util/certmanager/factory/factory_test.go | 50 +- pkg/util/certmanager/pki.go | 32 +- pkg/util/certmanager/pki_test.go | 4 +- pkg/yurthub/certificate/hubself/cert_mgr.go | 634 ------------------ .../certificate/hubself/cert_mgr_test.go | 291 -------- .../certificate/hubself/fake_cert_mgr.go | 155 ----- .../{interfaces => }/interfaces.go | 19 +- pkg/yurthub/certificate/token/testdata/ca.crt | 18 + pkg/yurthub/certificate/token/testdata/ca.key | 27 + .../certificate/token/testdata/fake_client.go | 542 +++++++++++++++ pkg/yurthub/certificate/token/token.go | 447 ++++++++++++ pkg/yurthub/certificate/token/token_test.go | 314 +++++++++ pkg/yurthub/healthchecker/fake_checker.go | 10 + pkg/yurthub/healthchecker/health_checker.go | 10 + pkg/yurthub/healthchecker/interfaces.go | 1 + pkg/yurthub/kubernetes/rest/config.go | 62 +- pkg/yurthub/kubernetes/rest/config_test.go | 172 ++--- pkg/yurthub/otaupdate/ota_test.go | 8 +- pkg/yurthub/server/certificate.go | 7 +- pkg/yurthub/server/certificate_test.go | 52 +- pkg/yurthub/server/nonresource_test.go | 9 +- pkg/yurthub/server/server.go | 57 +- pkg/yurthub/transport/transport.go | 66 +- pkg/yurthub/util/util.go | 116 +--- pkg/yurthub/util/util_test.go | 205 ------ 33 files changed, 1627 insertions(+), 1778 deletions(-) delete mode 100644 pkg/yurthub/certificate/hubself/cert_mgr.go delete mode 100644 pkg/yurthub/certificate/hubself/cert_mgr_test.go delete mode 100644 pkg/yurthub/certificate/hubself/fake_cert_mgr.go rename pkg/yurthub/certificate/{interfaces => }/interfaces.go (69%) create mode 100644 pkg/yurthub/certificate/token/testdata/ca.crt create mode 100644 pkg/yurthub/certificate/token/testdata/ca.key create mode 100644 pkg/yurthub/certificate/token/testdata/fake_client.go create mode 100644 pkg/yurthub/certificate/token/token.go create mode 100644 pkg/yurthub/certificate/token/token_test.go diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go index e2442a8ea28..d8db0130d79 100644 --- a/cmd/yurt-tunnel-server/app/start.go +++ b/cmd/yurt-tunnel-server/app/start.go @@ -177,12 +177,12 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error { }, stopCh) // 6. generate the TLS configuration based on the latest certificate - tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert, "server") + tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(serverCertMgr.Current, cfg.RootCert, "server") if err != nil { return err } - proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(tunnelProxyCertMgr, cfg.RootCert, "client") + proxyClientTlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(tunnelProxyCertMgr.Current, cfg.RootCert, "client") if err != nil { return err } diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index c2e2e6ece7a..d25e995002e 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -65,9 +65,6 @@ type YurtHubConfiguration struct { YurtHubProxyServerDummyAddr string YurtHubProxyServerSecureDummyAddr string GCFrequency int - CertMgrMode string - KubeletRootCAFilePath string - KubeletPairFilePath string NodeName string HeartbeatFailedRetry int HeartbeatHealthyThreshold int @@ -92,6 +89,7 @@ type YurtHubConfiguration struct { CertIPs []net.IP CoordinatorServer *url.URL MinRequestTimeout time.Duration + CaCertHashes []string } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -156,8 +154,6 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { YurtHubProxyServerDummyAddr: proxyServerDummyAddr, YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr, GCFrequency: options.GCFrequency, - KubeletRootCAFilePath: options.KubeletRootCAFilePath, - KubeletPairFilePath: options.KubeletPairFilePath, NodeName: options.NodeName, HeartbeatFailedRetry: options.HeartbeatFailedRetry, HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, @@ -180,6 +176,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { FilterManager: filterManager, CertIPs: certIPs, MinRequestTimeout: options.MinRequestTimeout, + CaCertHashes: options.CACertHashes, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index 171989c91f9..ded7e48b767 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -48,8 +48,6 @@ type YurtHubOptions struct { YurtHubProxySecurePort string GCFrequency int YurtHubCertOrganizations string - KubeletRootCAFilePath string - KubeletPairFilePath string NodeName string NodePoolName string LBMode string @@ -74,6 +72,8 @@ type YurtHubOptions struct { KubeletHealthGracePeriod time.Duration EnableNodePool bool MinRequestTimeout time.Duration + CACertHashes []string + UnsafeSkipCAVerification bool } // NewYurtHubOptions creates a new YurtHubOptions with a default config. @@ -85,8 +85,6 @@ func NewYurtHubOptions() *YurtHubOptions { YurtHubPort: util.YurtHubPort, YurtHubProxySecurePort: util.YurtHubProxySecurePort, GCFrequency: 120, - KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath, - KubeletPairFilePath: util.DefaultKubeletPairFilePath, LBMode: "rr", HeartbeatFailedRetry: 3, HeartbeatHealthyThreshold: 2, @@ -106,6 +104,7 @@ func NewYurtHubOptions() *YurtHubOptions { KubeletHealthGracePeriod: time.Second * 40, EnableNodePool: true, MinRequestTimeout: time.Second * 1800, + UnsafeSkipCAVerification: true, } return o } @@ -132,6 +131,10 @@ func (options *YurtHubOptions) Validate() error { return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err) } + if len(options.CACertHashes) == 0 && !options.UnsafeSkipCAVerification { + return fmt.Errorf("Set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue") + } + return nil } @@ -144,8 +147,6 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver") fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") fs.StringVar(&o.YurtHubCertOrganizations, "hub-cert-organizations", o.YurtHubCertOrganizations, "Organizations that will be added into hub's client certificate in hubself cert-mgr-mode, the format is: certOrg1,certOrg1,...") - fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.") - fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.") fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).") fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent") fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)") @@ -171,6 +172,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.KubeletHealthGracePeriod, "kubelet-health-grace-period", o.KubeletHealthGracePeriod, "the amount of time which we allow kubelet to be unresponsive before stop renew node lease") fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)") fs.DurationVar(&o.MinRequestTimeout, "min-request-timeout", o.MinRequestTimeout, "An optional field indicating at least how long a proxy handler must keep a request open before timing it out. Currently only honored by the local watch request handler(use request parameter timeoutSeconds firstly), which picks a randomized value above this number as the connection timeout, to spread out load.") + fs.StringSliceVar(&o.CACertHashes, "discovery-token-ca-cert-hash", []string{}, "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").") + fs.BoolVar(&o.UnsafeSkipCAVerification, "discovery-token-unsafe-skip-ca-verification", o.UnsafeSkipCAVerification, "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.") } // verifyDummyIP verify the specified ip is valid or not and set the default ip if empty diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 5b9a7308354..018cb1986d8 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -19,7 +19,6 @@ package app import ( "fmt" "net/url" - "path/filepath" "time" "github.com/spf13/cobra" @@ -33,7 +32,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" hubrest "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" @@ -87,24 +86,23 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command { func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { trace := 1 klog.Infof("%d. register cert managers", trace) - certManager, err := hubself.NewYurtHubCertManager(cfg) + certManager, err := token.NewYurtHubCertManager(nil, cfg, stopCh) if err != nil { return fmt.Errorf("failed to create cert manager for yurthub, %v", err) } trace++ certManager.Start() + defer certManager.Stop() err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { - curr := certManager.Current() - if curr != nil { + isReady := certManager.Ready() + if isReady { return true, nil } - - klog.Infof("waiting for preparing client certificate") return false, nil }) if err != nil { - return fmt.Errorf("client certificate preparation failed, %v", err) + return fmt.Errorf("hub certificates preparation failed, %v", err) } trace++ @@ -137,16 +135,15 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { } trace++ - klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode) - restConfigMgr, err := hubrest.NewRestConfigManager(cfg, certManager, healthChecker) + klog.Infof("%d. new restConfig manager", trace) + restConfigMgr, err := hubrest.NewRestConfigManager(certManager, healthChecker) if err != nil { return fmt.Errorf("could not new restConfig manager, %w", err) } trace++ klog.Infof("%d. create tls config for secure servers ", trace) - cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig( - restConfigMgr, certManager, filepath.Join(cfg.RootDir, "pki"), cfg.NodeName, cfg.CertIPs, stopCh) + cfg.TLSConfig, err = server.GenUseCertMgrAndTLSConfig(certManager) if err != nil { return fmt.Errorf("could not create tls config, %w", err) } diff --git a/pkg/controller/certificates/csrapprover.go b/pkg/controller/certificates/csrapprover.go index c1a73352ae8..534af79e4ce 100644 --- a/pkg/controller/certificates/csrapprover.go +++ b/pkg/controller/certificates/csrapprover.go @@ -40,7 +40,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -432,7 +432,7 @@ func isYurtHubNodeCert(csr *certificatesv1.CertificateSigningRequest, x509cr *x5 return false } else { for _, org := range x509cr.Subject.Organization { - if org != hubself.YurtHubCSROrg && org != user.NodesGroup && !strings.HasPrefix(org, yurtHubNodeCertOrgPrefix) { + if org != token.YurtHubCSROrg && org != user.NodesGroup && !strings.HasPrefix(org, yurtHubNodeCertOrgPrefix) { return false } } diff --git a/pkg/controller/certificates/csrapprover_test.go b/pkg/controller/certificates/csrapprover_test.go index 9ee97238e1f..85ac809c6a2 100644 --- a/pkg/controller/certificates/csrapprover_test.go +++ b/pkg/controller/certificates/csrapprover_test.go @@ -31,7 +31,7 @@ import ( "k8s.io/client-go/util/cert" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -91,7 +91,7 @@ func TestIsYurtCSR(t *testing.T) { certificatesv1.UsageKeyEncipherment, certificatesv1.UsageClientAuth, }, - Request: newCSRData("system:node:xxx", []string{hubself.YurtHubCSROrg, user.NodesGroup, "openyurt:tenant:xxx"}, []string{}, []net.IP{}), + Request: newCSRData("system:node:xxx", []string{token.YurtHubCSROrg, user.NodesGroup, "openyurt:tenant:xxx"}, []string{}, []net.IP{}), }, }, exp: true, @@ -106,7 +106,7 @@ func TestIsYurtCSR(t *testing.T) { certificatesv1.UsageKeyEncipherment, certificatesv1.UsageClientAuth, }, - Request: newCSRData("system:node:xxx", []string{hubself.YurtHubCSROrg, user.NodesGroup, "unknown org"}, []string{}, []net.IP{}), + Request: newCSRData("system:node:xxx", []string{token.YurtHubCSROrg, user.NodesGroup, "unknown org"}, []string{}, []net.IP{}), }, }, exp: false, diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index 782704b9823..f363deda7d1 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -30,10 +30,11 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/templates" - constants "github.com/openyurtio/openyurt/pkg/yurtadm/constants" + "github.com/openyurtio/openyurt/pkg/yurtadm/constants" enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -150,7 +151,7 @@ func getYurthubYaml(podManifestPath string) string { } func getYurthubConf() string { - return filepath.Join(hubself.HubRootDir, hubself.HubName) + return filepath.Join(token.DefaultRootDir, projectinfo.GetHubName()) } func getYurthubCacheDir() string { diff --git a/pkg/util/certmanager/factory/factory.go b/pkg/util/certmanager/factory/factory.go index e151d54be5f..c747b569092 100644 --- a/pkg/util/certmanager/factory/factory.go +++ b/pkg/util/certmanager/factory/factory.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/util/certificate" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/util" "github.com/openyurtio/openyurt/pkg/util/certmanager/store" ) @@ -78,20 +79,33 @@ type CertManagerFactory interface { New(*CertManagerConfig) (certificate.Manager, error) } +type factory struct { + clientsetFn certificate.ClientsetFunc + fileStore certificate.FileStore +} + func NewCertManagerFactory(clientSet kubernetes.Interface) CertManagerFactory { return &factory{ - clientset: clientSet, + clientsetFn: func(current *tls.Certificate) (kubernetes.Interface, error) { + return clientSet, nil + }, } } -type factory struct { - clientset kubernetes.Interface +func NewCertManagerFactoryWithFnAndStore(clientsetFn certificate.ClientsetFunc, store certificate.FileStore) CertManagerFactory { + return &factory{ + clientsetFn: clientsetFn, + fileStore: store, + } } func (f *factory) New(cfg *CertManagerConfig) (certificate.Manager, error) { - store, err := store.NewFileStoreWrapper(cfg.ComponentName, cfg.CertDir, cfg.CertDir, "", "") - if err != nil { - return nil, fmt.Errorf("failed to initialize the server certificate store: %w", err) + var err error + if util.IsNil(f.fileStore) { + f.fileStore, err = store.NewFileStoreWrapper(cfg.ComponentName, cfg.CertDir, cfg.CertDir, "", "") + if err != nil { + return nil, fmt.Errorf("failed to initialize the server certificate store: %w", err) + } } ips, dnsNames := cfg.IPs, cfg.DNSNames @@ -139,12 +153,11 @@ func (f *factory) New(cfg *CertManagerConfig) (certificate.Manager, error) { } return certificate.NewManager(&certificate.Config{ - ClientsetFn: func(current *tls.Certificate) (kubernetes.Interface, error) { - return f.clientset, nil - }, + ClientsetFn: f.clientsetFn, SignerName: cfg.SignerName, GetTemplate: getTemplate, Usages: usages, - CertificateStore: store, + CertificateStore: f.fileStore, + Logf: klog.Infof, }) } diff --git a/pkg/util/certmanager/factory/factory_test.go b/pkg/util/certmanager/factory/factory_test.go index 44a67885c2e..b83396873fa 100644 --- a/pkg/util/certmanager/factory/factory_test.go +++ b/pkg/util/certmanager/factory/factory_test.go @@ -17,6 +17,7 @@ limitations under the License. package factory import ( + "crypto/tls" "fmt" "net" "reflect" @@ -28,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/certmanager/store" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) @@ -36,40 +38,20 @@ const ( succeed = "\u2713" ) -var cs = &fake.Clientset{} -var fc = &factory{ - clientset: cs, -} - -func TestNewCertManagerFactory(t *testing.T) { - cs := &fake.Clientset{} - tests := []struct { - name string - clientSet kubernetes.Interface - expect CertManagerFactory - }{ - { - "normal", - cs, - fc, - }, +func TestNewCertManagerFactoryWithFnAndStore(t *testing.T) { + csFn := func(current *tls.Certificate) (kubernetes.Interface, error) { + return &fake.Clientset{}, nil + } + store, err := store.NewFileStoreWrapper(projectinfo.GetHubName(), "/var/lib", "/var/lib", "", "") + if err != nil { + t.Errorf("failed to new file store, %v", err) + return } - for _, tt := range tests { - tf := func(t *testing.T) { - t.Parallel() - t.Logf("\tTestCase: %s", tt.name) - { - get := NewCertManagerFactory(cs) - - if !reflect.DeepEqual(get, tt.expect) { - t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) - } - t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) + cmf := NewCertManagerFactoryWithFnAndStore(csFn, store) - } - } - t.Run(tt.name, tf) + if _, ok := cmf.(CertManagerFactory); !ok { + t.Errorf("expect CertManagerFactory object, but got %v", cmf) } } @@ -100,10 +82,11 @@ func TestNew(t *testing.T) { } for _, tt := range tests { - tf := func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", tt.name) { + fc := NewCertManagerFactory(&fake.Clientset{}) _, get := fc.New(tt.cfg) if !reflect.DeepEqual(get, tt.expect) { @@ -112,7 +95,6 @@ func TestNew(t *testing.T) { t.Logf("\t%s\texpect %v, get %v", succeed, tt.expect, get) } - } - t.Run(tt.name, tf) + }) } } diff --git a/pkg/util/certmanager/pki.go b/pkg/util/certmanager/pki.go index eb39df5f7f1..d114c8e716f 100644 --- a/pkg/util/certmanager/pki.go +++ b/pkg/util/certmanager/pki.go @@ -28,10 +28,10 @@ import ( "k8s.io/client-go/util/certificate" ) -// GenTLSConfigUseCertMgrAndCertPool generates a TLS configuration -// using the given certificate manager and x509 CertPool -func GenTLSConfigUseCertMgrAndCertPool( - m certificate.Manager, +// GenTLSConfigUseCurrentCertAndCertPool generates a TLS configuration +// using the given current certificate and x509 CertPool +func GenTLSConfigUseCurrentCertAndCertPool( + current func() *tls.Certificate, root *x509.CertPool, mode string) (*tls.Config, error) { tlsConfig := &tls.Config{ @@ -45,21 +45,25 @@ func GenTLSConfigUseCertMgrAndCertPool( case "server": tlsConfig.ClientCAs = root tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven - tlsConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cert := m.Current() - if cert == nil { - return &tls.Certificate{Certificate: nil}, nil + if current != nil { + tlsConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := current() + if cert == nil { + return &tls.Certificate{Certificate: nil}, nil + } + return cert, nil } - return cert, nil } case "client": tlsConfig.RootCAs = root - tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - cert := m.Current() - if cert == nil { - return &tls.Certificate{Certificate: nil}, nil + if current != nil { + tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert := current() + if cert == nil { + return &tls.Certificate{Certificate: nil}, nil + } + return cert, nil } - return cert, nil } default: return nil, fmt.Errorf("unsupported cert manager mode(only server or client), %s", mode) diff --git a/pkg/util/certmanager/pki_test.go b/pkg/util/certmanager/pki_test.go index 2b69048173c..800fac3c22f 100644 --- a/pkg/util/certmanager/pki_test.go +++ b/pkg/util/certmanager/pki_test.go @@ -174,7 +174,7 @@ func (s *fakeStore) Update(certPEM, keyPEM []byte) (*tls.Certificate, error) { return s.cert, nil } -func TestGenTLSConfigUseCertMgrAndCertPool(t *testing.T) { +func TestGenTLSConfigUseCurrentCertAndCertPool(t *testing.T) { store := &fakeStore{ cert: storeCertData.certificate, } @@ -222,7 +222,7 @@ func TestGenTLSConfigUseCertMgrAndCertPool(t *testing.T) { t.Parallel() t.Logf("\tTestCase: %s", tt.name) { - _, get := GenTLSConfigUseCertMgrAndCertPool(tt.m, tt.root, tt.mode) + _, get := GenTLSConfigUseCurrentCertAndCertPool(tt.m.Current, tt.root, tt.mode) if !reflect.DeepEqual(get, tt.expect) { t.Fatalf("\t%s\texpect %v, but get %v", failed, tt.expect, get) diff --git a/pkg/yurthub/certificate/hubself/cert_mgr.go b/pkg/yurthub/certificate/hubself/cert_mgr.go deleted file mode 100644 index 25b8f910cbf..00000000000 --- a/pkg/yurthub/certificate/hubself/cert_mgr.go +++ /dev/null @@ -1,634 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hubself - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "time" - - certificatesv1 "k8s.io/api/certificates/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/authentication/user" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/certificate" - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/util/certmanager/store" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" - "github.com/openyurtio/openyurt/pkg/yurthub/util" - "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" -) - -const ( - HubName = "yurthub" - HubRootDir = "/var/lib/" - YurtHubCSROrg = "openyurt:yurthub" - hubPkiDirName = "pki" - hubCaFileName = "ca.crt" - hubConfigFileName = "%s.conf" - bootstrapConfigFileName = "bootstrap-hub.conf" - bootstrapUser = "token-bootstrap-client" - defaultClusterName = "kubernetes" - clusterInfoName = "cluster-info" - kubeconfigName = "kubeconfig" -) - -type yurtHubCertManager struct { - remoteServers []*url.URL - hubCertOrganizations []string - bootstrapConfStore fs.FileSystemOperator - hubClientCertManager certificate.Manager - hubClientCertPath string - joinToken string - caFile string - nodeName string - rootDir string - hubName string - kubeletRootCAFilePath string - kubeletPairFilePath string - dialer *util.Dialer - stopCh chan struct{} -} - -// NewYurtHubCertManager new a YurtCertificateManager instance -func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) { - if cfg == nil || len(cfg.NodeName) == 0 || len(cfg.RemoteServers) == 0 { - return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager") - } - - hn := projectinfo.GetHubName() - if len(hn) == 0 { - hn = HubName - } - - rootDir := cfg.RootDir - if len(rootDir) == 0 { - rootDir = filepath.Join(HubRootDir, hn) - } - - ycm := &yurtHubCertManager{ - remoteServers: cfg.RemoteServers, - hubCertOrganizations: cfg.YurtHubCertOrganizations, - nodeName: cfg.NodeName, - joinToken: cfg.JoinToken, - kubeletRootCAFilePath: cfg.KubeletRootCAFilePath, - kubeletPairFilePath: cfg.KubeletPairFilePath, - rootDir: rootDir, - hubName: hn, - dialer: util.NewDialer("hub certificate manager"), - stopCh: make(chan struct{}), - } - - return ycm, nil -} - -func removeDirContents(dir string) error { - files, err := os.ReadDir(dir) - if err != nil { - return err - } - for _, d := range files { - err = os.RemoveAll(filepath.Join(dir, d.Name())) - if err != nil { - return err - } - } - return nil -} - -func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup() { - nServer := ycm.remoteServers[0].String() - - bcf := ycm.getBootstrapConfFile() - if existed, _ := util.FileExists(bcf); existed { - curKubeConfig, err := util.LoadKubeConfig(bcf) - if err == nil && curKubeConfig != nil { - oServer := curKubeConfig.Clusters[defaultClusterName].Server - if nServer == oServer { - klog.Infof("apiServer name %s not changed", oServer) - return - } else { - klog.Infof("config for apiServer %s found, need to recycle for new server %s", oServer, nServer) - } - } - } - - klog.Infof("clean up any stale files") - removeDirContents(ycm.rootDir) -} - -// Start init certificate manager and certs for hub agent -func (ycm *yurtHubCertManager) Start() { - // 0. verify, cleanup if needed - ycm.verifyServerAddrOrCleanup() - - // 1. create ca file for hub certificate manager - err := ycm.initCaCert() - if err != nil { - klog.Errorf("failed to init ca cert, %v", err) - return - } - klog.Infof("use %s ca file to bootstrap %s", ycm.caFile, ycm.hubName) - - // 2. create bootstrap config file for hub certificate manager - err = ycm.initBootstrap() - if err != nil { - klog.Errorf("failed to init bootstrap %v", err) - return - } - - // 3. create client certificate manager for hub certificate manager - err = ycm.initClientCertificateManager() - if err != nil { - klog.Errorf("failed to init client cert manager, %v", err) - return - } - - // 4. create hub config file - err = ycm.initHubConf() - if err != nil { - klog.Errorf("failed to init hub config, %v", err) - return - } -} - -// Stop the cert manager loop -func (ycm *yurtHubCertManager) Stop() { - if ycm.hubClientCertManager != nil { - ycm.hubClientCertManager.Stop() - } -} - -// Current returns the currently selected certificate from the certificate manager -func (ycm *yurtHubCertManager) Current() *tls.Certificate { - if ycm.hubClientCertManager != nil { - return ycm.hubClientCertManager.Current() - } - - return nil -} - -// ServerHealthy returns true if the cert manager believes the server is currently alive. -func (ycm *yurtHubCertManager) ServerHealthy() bool { - if ycm.hubClientCertManager != nil { - return ycm.hubClientCertManager.ServerHealthy() - } - - return false -} - -// Update update bootstrap conf file by new bearer token. -func (ycm *yurtHubCertManager) Update(cfg *config.YurtHubConfiguration) error { - if cfg == nil { - return nil - } - - err := ycm.updateBootstrapConfFile(cfg.JoinToken) - if err != nil { - klog.Errorf("could not update hub agent bootstrap config file, %v", err) - return err - } - - return nil -} - -// GetCaFile returns the path of ca file -func (ycm *yurtHubCertManager) GetCaFile() string { - return ycm.caFile -} - -// GetConfFilePath returns the path of yurtHub config file path -func (ycm *yurtHubCertManager) GetConfFilePath() string { - return ycm.getHubConfFile() -} - -// NotExpired returns hub client cert is expired or not. -// True: not expired -// False: expired -func (ycm *yurtHubCertManager) NotExpired() bool { - return ycm.Current() != nil -} - -// initCaCert create ca file for hub certificate manager -func (ycm *yurtHubCertManager) initCaCert() error { - caFile := ycm.getCaFile() - ycm.caFile = caFile - caExisted := false - - if exists, err := util.FileExists(caFile); exists { - caExisted = true - klog.Infof("%s file already exists, check with server", caFile) - } else if err != nil { - klog.Errorf("could not stat ca file %s, %v", caFile, err) - return err - } else { - klog.Infof("%s file not exists, so create it", caFile) - } - - insecureRestConfig, err := createInsecureRestClientConfig(ycm.remoteServers[0]) - if err != nil { - klog.Errorf("could not create insecure rest config, %v", err) - return err - } - - insecureClient, err := clientset.NewForConfig(insecureRestConfig) - if err != nil { - klog.Errorf("could not new insecure client, %v", err) - return err - } - - // make sure configMap kube-public/cluster-info in k8s cluster beforehand - insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), clusterInfoName, metav1.GetOptions{}) - if err != nil { - if caExisted { - klog.Errorf("couldn't reach server, use existed %s file", caFile) - return nil - } - klog.Errorf("failed to get cluster-info configmap, %v", err) - return err - } - - kubeconfigStr, ok := insecureClusterInfo.Data[kubeconfigName] - if !ok || len(kubeconfigStr) == 0 { - return fmt.Errorf("no kubeconfig in cluster-info configmap of kube-public namespace") - } - - kubeConfig, err := clientcmd.Load([]byte(kubeconfigStr)) - if err != nil { - return fmt.Errorf("could not load kube config string, %w", err) - } - - if len(kubeConfig.Clusters) != 1 { - return fmt.Errorf("more than one cluster setting in cluster-info configmap") - } - - var clusterCABytes []byte - for _, cluster := range kubeConfig.Clusters { - clusterCABytes = cluster.CertificateAuthorityData - } - - if caExisted { - var curCABytes []byte - if curCABytes, err = os.ReadFile(caFile); err != nil { - klog.Infof("could not read existed %s file, %v, ", caFile, err) - } - - if bytes.Equal(clusterCABytes, curCABytes) { - klog.Infof("%s file matched with server's, reuse it", caFile) - return nil - } else { - klog.Infof("%s file is outdated, need to create a new one", caFile) - removeDirContents(ycm.rootDir) - } - } - - if err := certutil.WriteCert(caFile, clusterCABytes); err != nil { - klog.Errorf("could not write %s ca cert, %v", ycm.hubName, err) - return err - } - - return nil -} - -// initBootstrap create bootstrap config file for hub certificate manager -func (ycm *yurtHubCertManager) initBootstrap() error { - ycm.bootstrapConfStore = fs.FileSystemOperator{} - bootstrapConfigFilePath := filepath.Join(ycm.rootDir, bootstrapConfigFileName) - contents, err := ycm.bootstrapConfStore.Read(bootstrapConfigFilePath) - if err == fs.ErrNotExists || len(contents) == 0 { - klog.Infof("%s bootstrap conf file does not exist or its content is empty, so create it", ycm.hubName) - return ycm.createBootstrapConfFile(ycm.joinToken) - } else if err != nil { - klog.Infof("could not get bootstrap conf file, %v", err) - return err - } else { - klog.Infof("%s bootstrap conf file already exists, skip init bootstrap", ycm.hubName) - return nil - } -} - -// initClientCertificateManager init hub client certificate manager -func (ycm *yurtHubCertManager) initClientCertificateManager() error { - s, err := store.NewFileStoreWrapper(ycm.hubName, ycm.getPkiDir(), ycm.getPkiDir(), "", "") - if err != nil { - klog.Errorf("failed to init %s client cert store, %v", ycm.hubName, err) - return err - - } - ycm.hubClientCertPath = s.CurrentPath() - - orgs := []string{YurtHubCSROrg, user.NodesGroup} - if len(ycm.hubCertOrganizations) > 0 { - for _, v := range ycm.hubCertOrganizations { - if v != YurtHubCSROrg && v != user.NodesGroup { - orgs = append(orgs, v) - } - } - } - - m, err := certificate.NewManager(&certificate.Config{ - ClientsetFn: ycm.generateCertClientFn, - SignerName: certificatesv1.KubeAPIServerClientSignerName, - Template: &x509.CertificateRequest{ - Subject: pkix.Name{ - CommonName: fmt.Sprintf("system:node:%s", ycm.nodeName), - Organization: orgs, - }, - }, - Usages: []certificatesv1.KeyUsage{ - certificatesv1.UsageDigitalSignature, - certificatesv1.UsageKeyEncipherment, - certificatesv1.UsageClientAuth, - }, - - CertificateStore: s, - }) - if err != nil { - return fmt.Errorf("failed to initialize client certificate manager: %w", err) - } - ycm.hubClientCertManager = m - m.Start() - - return nil -} - -// getBootstrapClientConfig get rest client config from bootstrap conf file. -// and when no bearer token in bootstrap conf file, kubelet.conf will be used instead. -func (ycm *yurtHubCertManager) getBootstrapClientConfig(healthyServer *url.URL) (*restclient.Config, error) { - restCfg, err := util.LoadRESTClientConfig(ycm.getBootstrapConfFile()) - if err != nil { - klog.Errorf("could not load rest client config from bootstrap file(%s), %v", ycm.getBootstrapConfFile(), err) - return nil, err - } - - if len(restCfg.BearerToken) != 0 { - klog.V(3).Infof("join token is set for bootstrap client config") - // re-fix healthy host for bootstrap client config - restCfg.Host = healthyServer.String() - return restCfg, nil - } - - klog.Infof("no join token, so use kubelet config to bootstrap hub") - // use kubelet.conf to bootstrap hub agent - return util.LoadKubeletRestClientConfig(healthyServer, ycm.kubeletRootCAFilePath, ycm.kubeletPairFilePath) -} - -func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { - var cfg *restclient.Config - var healthyServer *url.URL - hubConfFile := ycm.getHubConfFile() - - _ = wait.PollInfinite(30*time.Second, func() (bool, error) { - healthyServer = ycm.remoteServers[0] - if healthyServer == nil { - klog.V(3).Infof("all of remote servers are unhealthy, just wait") - return false, nil - } - - // If we have a valid certificate, use that to fetch CSRs. - // Otherwise use the bootstrap conf file. - if current != nil { - klog.V(3).Infof("use %s config to create csr client", ycm.hubName) - // use the valid certificate - kubeConfig, err := util.LoadRESTClientConfig(hubConfFile) - if err != nil { - klog.Errorf("could not load %s kube config, %v", ycm.hubName, err) - return false, nil - } - - // re-fix healthy host for cert manager - kubeConfig.Host = healthyServer.String() - cfg = kubeConfig - } else { - klog.V(3).Infof("use bootstrap client config to create csr client") - // bootstrap is updated - bootstrapClientConfig, err := ycm.getBootstrapClientConfig(healthyServer) - if err != nil { - klog.Errorf("could not load bootstrap config in clientFn, %v", err) - return false, nil - } - - cfg = bootstrapClientConfig - } - - if cfg != nil { - klog.V(3).Infof("bootstrap client config: %#+v", cfg) - // re-fix dial for conn management - cfg.Dial = ycm.dialer.DialContext - } - return true, nil - }) - - // avoid tcp conn leak: certificate rotated, so close old tcp conn that used to rotate certificate - klog.V(2).Infof("avoid tcp conn leak, close old tcp conn that used to rotate certificate") - ycm.dialer.Close(strings.Trim(cfg.Host, "https://")) - - return clientset.NewForConfig(cfg) -} - -// initHubConf init hub agent conf file. -func (ycm *yurtHubCertManager) initHubConf() error { - hubConfFile := ycm.getHubConfFile() - if exists, err := util.FileExists(hubConfFile); exists { - klog.Infof("%s config file already exists, skip init config file", ycm.hubName) - return nil - } else if err != nil { - klog.Errorf("could not stat %s config file %s, %v", ycm.hubName, hubConfFile, err) - return err - } else { - klog.Infof("%s file not exists, so create it", hubConfFile) - } - - bootstrapClientConfig, err := util.LoadRESTClientConfig(ycm.getBootstrapConfFile()) - if err != nil { - klog.Errorf("could not load bootstrap client config for init cert store, %v", err) - return err - } - hubClientConfig := restclient.AnonymousClientConfig(bootstrapClientConfig) - hubClientConfig.KeyFile = ycm.hubClientCertPath - hubClientConfig.CertFile = ycm.hubClientCertPath - err = util.CreateKubeConfigFile(hubClientConfig, hubConfFile) - if err != nil { - klog.Errorf("could not create %s config file, %v", ycm.hubName, err) - return err - } - - return nil -} - -// getPkiDir returns the directory for storing hub agent pki -func (ycm *yurtHubCertManager) getPkiDir() string { - return filepath.Join(ycm.rootDir, hubPkiDirName) -} - -// getCaFile returns the path of ca file -func (ycm *yurtHubCertManager) getCaFile() string { - return filepath.Join(ycm.getPkiDir(), hubCaFileName) -} - -// getBootstrapConfFile returns the path of bootstrap conf file -func (ycm *yurtHubCertManager) getBootstrapConfFile() string { - return filepath.Join(ycm.rootDir, bootstrapConfigFileName) -} - -// getHubConfFile returns the path of hub agent conf file. -func (ycm *yurtHubCertManager) getHubConfFile() string { - return filepath.Join(ycm.rootDir, fmt.Sprintf(hubConfigFileName, ycm.hubName)) -} - -// createBasic create basic client cmd config -func createBasic(apiServerAddr string, caCert []byte) *clientcmdapi.Config { - contextName := fmt.Sprintf("%s@%s", bootstrapUser, defaultClusterName) - - return &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - defaultClusterName: { - Server: apiServerAddr, - CertificateAuthorityData: caCert, - }, - }, - Contexts: map[string]*clientcmdapi.Context{ - contextName: { - Cluster: defaultClusterName, - AuthInfo: bootstrapUser, - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{}, - CurrentContext: contextName, - } -} - -// createInsecureRestClientConfig create insecure rest client config. -func createInsecureRestClientConfig(remoteServer *url.URL) (*restclient.Config, error) { - if remoteServer == nil { - return nil, fmt.Errorf("no healthy remote server") - } - cfg := createBasic(remoteServer.String(), []byte{}) - cfg.Clusters[defaultClusterName].InsecureSkipTLSVerify = true - - restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return nil, fmt.Errorf("failed to create insecure rest client configuration, %w", err) - } - return restConfig, nil -} - -// createBootstrapConf create bootstrap conf info -func createBootstrapConf(apiServerAddr, caFile, joinToken string) *clientcmdapi.Config { - if len(apiServerAddr) == 0 || len(caFile) == 0 { - return nil - } - - exists, err := util.FileExists(caFile) - if err != nil || !exists { - klog.Errorf("ca file(%s) is not exist, %v", caFile, err) - return nil - } - - caCert, err := os.ReadFile(caFile) - if err != nil { - klog.Errorf("could not read ca file(%s), %v", caFile, err) - return nil - } - - cfg := createBasic(apiServerAddr, caCert) - cfg.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} - - return cfg -} - -// createBootstrapConfFile create bootstrap conf file -func (ycm *yurtHubCertManager) createBootstrapConfFile(joinToken string) error { - remoteServer := ycm.remoteServers[0] - if remoteServer == nil || len(remoteServer.Host) == 0 { - return fmt.Errorf("no healthy server for create bootstrap conf file") - } - - bootstrapConfig := createBootstrapConf(remoteServer.String(), ycm.caFile, joinToken) - if bootstrapConfig == nil { - return fmt.Errorf("could not create bootstrap config for %s", ycm.hubName) - } - - content, err := clientcmd.Write(*bootstrapConfig) - if err != nil { - return fmt.Errorf("could not create bootstrap config into bytes got error, %v", err) - } - - bootstrapConfigFilePath := filepath.Join(ycm.rootDir, bootstrapConfigFileName) - if err := ycm.bootstrapConfStore.DeleteFile(bootstrapConfigFilePath); err != nil { - return fmt.Errorf("failed to delete existing bootstrap file at %s, %v", bootstrapConfigFilePath, err) - } - if err := ycm.bootstrapConfStore.CreateFile(bootstrapConfigFilePath, content); err != nil { - return fmt.Errorf("could not create bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err) - } - return nil -} - -// updateBootstrapConfFile update bearer token in bootstrap conf file -func (ycm *yurtHubCertManager) updateBootstrapConfFile(joinToken string) error { - if len(joinToken) == 0 { - return fmt.Errorf("joinToken should not be empty when update bootstrap conf file") - } - - var curKubeConfig *clientcmdapi.Config - if existed, _ := util.FileExists(ycm.getBootstrapConfFile()); !existed { - klog.Infof("bootstrap conf file not exists(maybe deleted unintentionally), so create a new one") - return ycm.createBootstrapConfFile(joinToken) - } - - curKubeConfig, err := util.LoadKubeConfig(ycm.getBootstrapConfFile()) - if err != nil || curKubeConfig == nil { - klog.Errorf("could not get current bootstrap config for %s, %v", ycm.hubName, err) - return fmt.Errorf("could not load bootstrap conf file(%s), %w", ycm.getBootstrapConfFile(), err) - } - - if curKubeConfig.AuthInfos[bootstrapUser] != nil { - if curKubeConfig.AuthInfos[bootstrapUser].Token == joinToken { - klog.Infof("join token for %s bootstrap conf file is not changed", ycm.hubName) - return nil - } - } - - curKubeConfig.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken} - content, err := clientcmd.Write(*curKubeConfig) - if err != nil { - klog.Errorf("could not update bootstrap config into bytes, %v", err) - return err - } - - bootstrapConfigFilePath := filepath.Join(ycm.rootDir, bootstrapConfigFileName) - if err := ycm.bootstrapConfStore.Write(bootstrapConfigFilePath, content); err != nil { - return fmt.Errorf("failed to write bootstrap to file %s, %v", bootstrapConfigFilePath, err) - } - return nil -} diff --git a/pkg/yurthub/certificate/hubself/cert_mgr_test.go b/pkg/yurthub/certificate/hubself/cert_mgr_test.go deleted file mode 100644 index b7d415043c2..00000000000 --- a/pkg/yurthub/certificate/hubself/cert_mgr_test.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2020 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hubself - -import ( - "io/ioutil" - "net/url" - "os" - "testing" - - "k8s.io/client-go/util/certificate" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/yurthub/util" - "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" -) - -var ( - testCaCert = []byte(`-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJANXr+UzRFq4TMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwIBcNMTcwNDI2MjMyNzMyWhgPMjExNzA0 -MDIyMzI3MzJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAqvbkN4RShH1rL37JFp4fZPnn0JUhVWWsrP8NOomJ -pXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbMQ3ABJwIDAQABo1AwTjAdBgNV -HQ4EFgQUEGBc6YYheEZ/5MhwqSUYYPYRj2MwHwYDVR0jBBgwFoAUEGBc6YYheEZ/ -5MhwqSUYYPYRj2MwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAIyNmznk -5dgJY52FppEEcfQRdS5k4XFPc22SHPcz77AHf5oWZ1WG9VezOZZPp8NCiFDDlDL8 -yma33a5eMyTjLD8= ------END CERTIFICATE-----`) -) - -func TestNewYurtHubCertManager(t *testing.T) { - testUrl, err := url.Parse("https://127.0.0.1") - if err != nil { - t.Fatalf("parse test url failed. err:%v", err) - } - testNodeName := "localhost" - - type args struct { - cfg *config.YurtHubConfiguration - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no config", args{}, true}, - {"config has no node name", args{&config.YurtHubConfiguration{ - RemoteServers: []*url.URL{testUrl}, - }}, true}, - {"config has no remote server", args{&config.YurtHubConfiguration{ - NodeName: testNodeName, - }}, true}, - {"config valid", args{&config.YurtHubConfiguration{ - RemoteServers: []*url.URL{testUrl}, - NodeName: testNodeName, - }}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err = NewYurtHubCertManager(tt.args.cfg) - if (err != nil) != tt.wantErr { - t.Errorf("NewYurtHubCertManager() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func Test_removeDirContents(t *testing.T) { - - dir, err := ioutil.TempDir("", "yurthub-test-removeDirContents") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - - tempFile := dir + "/tmp.txt" - if err = ioutil.WriteFile(tempFile, nil, 0600); err != nil { - t.Fatalf("Unable to create the test file %q: %v", tempFile, err) - } - - type args struct { - dir string - } - tests := []struct { - name string - args args - file string - wantErr bool - }{ - {"no input dir", args{}, "", true}, - {"input dir exist", args{dir}, tempFile, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err = removeDirContents(tt.args.dir); (err != nil) != tt.wantErr { - t.Errorf("removeDirContents() error = %v, wantErr %v", err, tt.wantErr) - } - - if tt.file != "" { - if _, err = os.Stat(tt.file); err == nil || !os.IsNotExist(err) { - t.Errorf("after remote dir content, no file should exist. file:%s", tt.file) - } - } - }) - } -} - -func Test_createInsecureRestClientConfig(t *testing.T) { - testUrl, err := url.Parse("https://127.0.0.1") - if err != nil { - t.Fatalf("parse test url failed. err:%v", err) - } - type args struct { - remoteServer *url.URL - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no remote server", args{}, true}, - {"remote server exist", args{testUrl}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := createInsecureRestClientConfig(tt.args.remoteServer) - if (err != nil) != tt.wantErr { - t.Errorf("createInsecureRestClientConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func Test_yurtHubCertManager_createBootstrapConfFile(t *testing.T) { - testUrl, err := url.Parse("https://127.0.0.1") - if err != nil { - t.Fatalf("parse test url failed. err:%v", err) - } - testFakeStore := fs.FileSystemOperator{} - dir, err := ioutil.TempDir("", "yurthub-test-ca") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - testCAFile := dir + "/ca.crt" - if err = ioutil.WriteFile(testCAFile, testCaCert, 0600); err != nil { - t.Fatalf("Unable to create the test file %q: %v", testCAFile, err) - } - type fields struct { - remoteServers []*url.URL - hubCertOrganizations []string - bootstrapConfStore fs.FileSystemOperator - hubClientCertManager certificate.Manager - hubClientCertPath string - joinToken string - caFile string - nodeName string - rootDir string - hubName string - kubeletRootCAFilePath string - kubeletPairFilePath string - dialer *util.Dialer - stopCh chan struct{} - } - type args struct { - joinToken string - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - {"illegal remote server", fields{ - remoteServers: []*url.URL{{}}, - }, args{}, true}, - {"ca file not exist", fields{ - remoteServers: []*url.URL{testUrl}, - }, args{}, true}, - {"success", fields{ - remoteServers: []*url.URL{testUrl}, - bootstrapConfStore: testFakeStore, - caFile: testCAFile, - }, args{}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ycm := &yurtHubCertManager{ - remoteServers: tt.fields.remoteServers, - hubCertOrganizations: tt.fields.hubCertOrganizations, - bootstrapConfStore: tt.fields.bootstrapConfStore, - hubClientCertManager: tt.fields.hubClientCertManager, - hubClientCertPath: tt.fields.hubClientCertPath, - joinToken: tt.fields.joinToken, - caFile: tt.fields.caFile, - nodeName: tt.fields.nodeName, - rootDir: tt.fields.rootDir, - hubName: tt.fields.hubName, - kubeletRootCAFilePath: tt.fields.kubeletRootCAFilePath, - kubeletPairFilePath: tt.fields.kubeletPairFilePath, - dialer: tt.fields.dialer, - stopCh: tt.fields.stopCh, - } - if err = ycm.createBootstrapConfFile(tt.args.joinToken); (err != nil) != tt.wantErr { - t.Errorf("createBootstrapConfFile() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func Test_yurtHubCertManager_updateBootstrapConfFile(t *testing.T) { - type fields struct { - remoteServers []*url.URL - hubCertOrganizations []string - bootstrapConfStore fs.FileSystemOperator - hubClientCertManager certificate.Manager - hubClientCertPath string - joinToken string - caFile string - nodeName string - rootDir string - hubName string - kubeletRootCAFilePath string - kubeletPairFilePath string - dialer *util.Dialer - stopCh chan struct{} - } - type args struct { - joinToken string - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - {"joinToken not exist", fields{}, args{}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ycm := &yurtHubCertManager{ - remoteServers: tt.fields.remoteServers, - hubCertOrganizations: tt.fields.hubCertOrganizations, - bootstrapConfStore: tt.fields.bootstrapConfStore, - hubClientCertManager: tt.fields.hubClientCertManager, - hubClientCertPath: tt.fields.hubClientCertPath, - joinToken: tt.fields.joinToken, - caFile: tt.fields.caFile, - nodeName: tt.fields.nodeName, - rootDir: tt.fields.rootDir, - hubName: tt.fields.hubName, - kubeletRootCAFilePath: tt.fields.kubeletRootCAFilePath, - kubeletPairFilePath: tt.fields.kubeletPairFilePath, - dialer: tt.fields.dialer, - stopCh: tt.fields.stopCh, - } - if err := ycm.updateBootstrapConfFile(tt.args.joinToken); (err != nil) != tt.wantErr { - t.Errorf("updateBootstrapConfFile() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/pkg/yurthub/certificate/hubself/fake_cert_mgr.go b/pkg/yurthub/certificate/hubself/fake_cert_mgr.go deleted file mode 100644 index c814b2ba01e..00000000000 --- a/pkg/yurthub/certificate/hubself/fake_cert_mgr.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hubself - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "path/filepath" - - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" - "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" -) - -var ( - defaultCertificatePEM = `-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 -MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV -tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV -HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv -0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV -NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl -nkVA6wyOSDYBf3o= ------END CERTIFICATE-----` - defaultKeyPEM = `-----BEGIN RSA PRIVATE KEY----- -MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC -PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M -zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf -G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg -XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK -iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar -e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX -54LzHNk/+Q== ------END RSA PRIVATE KEY-----` -) - -type FakeYurtHubCertManager struct { - certificatePEM string - keyPEM string - rootDir string - hubName string - yurthubConifFile string - certStore fs.FileSystemOperator -} - -// NewFakeYurtHubCertManager new a YurtCertificateManager instance -func NewFakeYurtHubCertManager(rootDir, yurthubConfigFile, certificatePEM, keyPEM string) (interfaces.YurtCertificateManager, error) { - hn := projectinfo.GetHubName() - if len(hn) == 0 { - hn = HubName - } - if len(certificatePEM) == 0 { - certificatePEM = defaultCertificatePEM - } - if len(keyPEM) == 0 { - keyPEM = defaultKeyPEM - } - - rd := rootDir - if len(rd) == 0 { - rd = filepath.Join(HubRootDir, hn) - } - - fyc := &FakeYurtHubCertManager{ - certificatePEM: certificatePEM, - keyPEM: keyPEM, - rootDir: rd, - hubName: hn, - yurthubConifFile: yurthubConfigFile, - certStore: fs.FileSystemOperator{}, - } - - return fyc, nil -} - -// Start create the yurthub.conf file -func (fyc *FakeYurtHubCertManager) Start() { - fileName := fmt.Sprintf(hubConfigFileName, fyc.hubName) - yurthubConf := filepath.Join(fyc.rootDir, fileName) - if err := fyc.certStore.CreateFile(yurthubConf, []byte(fyc.yurthubConifFile)); err != nil { - klog.Errorf("Unable to create the file %s: %v", yurthubConf, err) - } -} - -// Stop do nothing -func (fyc *FakeYurtHubCertManager) Stop() {} - -// Current returns the certificate created by the entered fyc.certificatePEM and fyc.keyPEM -func (fyc *FakeYurtHubCertManager) Current() *tls.Certificate { - certificate, err := tls.X509KeyPair([]byte(fyc.certificatePEM), []byte(fyc.keyPEM)) - if err != nil { - panic(fmt.Sprintf("Unable to initialize certificate: %v", err)) - } - certs, err := x509.ParseCertificates(certificate.Certificate[0]) - if err != nil { - panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err)) - } - certificate.Leaf = certs[0] - - return &certificate -} - -// ServerHealthy returns true -func (fyc *FakeYurtHubCertManager) ServerHealthy() bool { - return true -} - -// Update do nothing -func (fyc *FakeYurtHubCertManager) Update(_ *config.YurtHubConfiguration) error { - return nil -} - -// GetCaFile returns the empty path -func (fyc *FakeYurtHubCertManager) GetCaFile() string { - return "" -} - -// GetConfFilePath returns the path of yurtHub config file path -func (fyc *FakeYurtHubCertManager) GetConfFilePath() string { - return fyc.getHubConfFile() -} - -// NotExpired returns true -func (fyc *FakeYurtHubCertManager) NotExpired() bool { - return fyc.Current() != nil -} - -// getHubConfFile returns the path of hub agent conf file. -func (fyc *FakeYurtHubCertManager) getHubConfFile() string { - return filepath.Join(fyc.rootDir, fmt.Sprintf(hubConfigFileName, fyc.hubName)) -} diff --git a/pkg/yurthub/certificate/interfaces/interfaces.go b/pkg/yurthub/certificate/interfaces.go similarity index 69% rename from pkg/yurthub/certificate/interfaces/interfaces.go rename to pkg/yurthub/certificate/interfaces.go index 8cb26ee94c5..6ac9d8a9bae 100644 --- a/pkg/yurthub/certificate/interfaces/interfaces.go +++ b/pkg/yurthub/certificate/interfaces.go @@ -14,19 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package interfaces +package certificate import ( - "k8s.io/client-go/util/certificate" - - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "crypto/tls" ) // YurtCertificateManager is responsible for managing node certificate for yurthub type YurtCertificateManager interface { - certificate.Manager - Update(cfg *config.YurtHubConfiguration) error - GetConfFilePath() string + Start() + Stop() + // Ready should be called after yurt certificate manager started by Start. + Ready() bool + UpdateBootstrapConf(joinToken string) error + GetHubConfFile() string GetCaFile() string - NotExpired() bool + GetAPIServerClientCert() *tls.Certificate + GetHubServerCert() *tls.Certificate + GetHubServerCertFile() string } diff --git a/pkg/yurthub/certificate/token/testdata/ca.crt b/pkg/yurthub/certificate/token/testdata/ca.crt new file mode 100644 index 00000000000..419f970016b --- /dev/null +++ b/pkg/yurthub/certificate/token/testdata/ca.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC9zCCAd+gAwIBAgIJAOWJ8tWNUIsZMA0GCSqGSIb3DQEBCwUAMBIxEDAOBgNV +BAMMB2t1YmUtY2EwHhcNMTYxMjIyMDAyNTI5WhcNNDQwNTA5MDAyNTI5WjASMRAw +DgYDVQQDDAdrdWJlLWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +1HK1d2p7N7UC6px8lVtABw8jPpVyNYjrJmI+TKTTdCgWGsUTFMCw4t4Q/KQDDlvB +P19uPhbfp8aLwOWXBCxOPZzlM2mAEjSUgKjbyGCW/8vaXa2VgQm3tKZdydKiFvIo +fEsNA+58w8A0WWEB8wYFcdCt8uPyQ0ws/TxE+WW3u7EPlC0/inIX9JqeZZMpDk3N +lHEv/pGEjQmoet/hBwGHq9PKepkN5/V6rrSADJ5I4Uklp2f7G9MCP/zV8xKfs0lK +CMoJsIPK3nL9N3C0rqBQPfcyKE2fnEkxC3UVZA8brvLTkBfOgmM2eVg/nauU1ejv +zOJL7tDwUioLriw2hiGrFwIDAQABo1AwTjAdBgNVHQ4EFgQUbGJxJeW7BgZ4xSmW +d3Aw3gq8YZUwHwYDVR0jBBgwFoAUbGJxJeW7BgZ4xSmWd3Aw3gq8YZUwDAYDVR0T +BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAunzpYAxpzguzxG83pK5n3ObsGDwO +78d38qX1VRvMLPvioZxYgquqqFPdLI3xe8b8KdZNzb65549tgjAI17tTKGTRgJu5 +yzLU1tO4vNaAFecMCtPvElYfkrAv2vbGCVJ1bYKTnjdu3083jG3sY9TDj0364A57 +lNwKEd5uxHGWg4H+NbyHkDqfKmllzLvJ9XjSWBPmNVLSW50hV+h9fUXgz9LN+qVY +VEDfAEWqb6PVy9ANw8A8QLnuSRxbd7hAigtlC4MwzYJ6tyFIIH6bCIgfoZuA+brm +WGcpIxl4fKEGafSgjsK/6Yhb61mkhHmG16mzEUZNkNsjiYJuF2QxpOlQrw== +-----END CERTIFICATE----- diff --git a/pkg/yurthub/certificate/token/testdata/ca.key b/pkg/yurthub/certificate/token/testdata/ca.key new file mode 100644 index 00000000000..fdf489609a8 --- /dev/null +++ b/pkg/yurthub/certificate/token/testdata/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA1HK1d2p7N7UC6px8lVtABw8jPpVyNYjrJmI+TKTTdCgWGsUT +FMCw4t4Q/KQDDlvBP19uPhbfp8aLwOWXBCxOPZzlM2mAEjSUgKjbyGCW/8vaXa2V +gQm3tKZdydKiFvIofEsNA+58w8A0WWEB8wYFcdCt8uPyQ0ws/TxE+WW3u7EPlC0/ +inIX9JqeZZMpDk3NlHEv/pGEjQmoet/hBwGHq9PKepkN5/V6rrSADJ5I4Uklp2f7 +G9MCP/zV8xKfs0lKCMoJsIPK3nL9N3C0rqBQPfcyKE2fnEkxC3UVZA8brvLTkBfO +gmM2eVg/nauU1ejvzOJL7tDwUioLriw2hiGrFwIDAQABAoIBAFJCmEFE2bEYRajS +LusmCgSxt9PjyfUwrtyN7dF/gODZJLX42QqQEe3GTo2EdCp7HLiNGwKvmKo+Fp76 +Rx82iJUSyyy9DPn/ogCvYWqU++LP7B2ZuOnd+WPZhzc+d8Sqv0JhTQjYrzaclaiG +B1syWalYRAJogMXOGR102MA4wovJrlHFuTVSWiDe0uguLxyjoTMIRqbib9ZAMSLX +bfcM2abGpXgq10abda3KKAJbZyr2fnBvqKTs4a4zYeHJpQT+NBPMiryb2WnPFg+b +93nrjDxUtPsx8NJz6HGkSQLagXkZX2J1JpT8loaNIdyQHab1LNXptc84LR8xxusy +bs5NowECgYEA+j+SwVgeC+NCUIfxr3F9zPAD9A0Tk3gD4z+j0opfLIMghX4jtK0e +9fQyglecAbojlkEUk/js5IVZ0IIhBNPWXxKtdShZO7EmJ6Z5IEmFrZK1xUomYBa2 +BfysqSAkxVLsTDIfI0Q4DHQNDOV+iY3j8WoaR51cXr+IY+mYBGSNI80CgYEA2VS5 +X5QHDxoh3r5ORiyab3ciubEofJ29D3NR1tCe9ZgSYRV5Y7T/4KPpZdpsEX/ydYD6 +X4DyURuYNK7PUR8DSlX7/VuMzHThqGJMaT0LE+alU4bruiad33X1WXgtcPTGCic0 +8il50TZTgba0CwxuCO1eVb3IijwgJBX/byM67nMCgYEA7As1KSwtwzbMoVtpa/xY +Fgu7HuOKuIn22M55fylH1puk/GXb1huJ3aNGVU2/+J0T3jFq8JxXDsJ90kA8Vupe +BXV/qceyS6yv+ax8Cilvbya4T+y+P9qMPR912V1Zccri2ohYeJJrb8uzV5vM/ICb +JmbXfP+AVlrBksSOwG37920CgYEAsSi2X6o8QtxLhdZd2ihbz8cu4G4AkezHhAO+ +T70KBytquAcYR+Xwu38CMEvn0jAZRh3YeueTH/i9jxx81STRutPysSni0Xvpwyg2 +H4dqM1PNqxQNrlXyVYlDciZb7HsrwHULXOfgbGG7mr6Db4o3XEGap4woID84+BGS +glcWn+8CgYEA36uulmZcodfet04qQvlDtr1d7mwLdTR/JAO0ZBIgFH7eGZdEVh8O +DoTJTdSSJGiv8J35PwEXfhKHjhgOjDocLYu+yCOwVj7jRdHqlDS1BaE36Hzdw0rb +mWkBRMGJtGhzhoRJEFHAnoLXc9danRfnHwVR58drlf7bjR5I9eU9u1I= +-----END RSA PRIVATE KEY----- diff --git a/pkg/yurthub/certificate/token/testdata/fake_client.go b/pkg/yurthub/certificate/token/testdata/fake_client.go new file mode 100644 index 00000000000..96e664b582a --- /dev/null +++ b/pkg/yurthub/certificate/token/testdata/fake_client.go @@ -0,0 +1,542 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testdata + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "fmt" + "math/big" + "path/filepath" + "sort" + "sync/atomic" + "time" + + "github.com/pkg/errors" + certificatesv1 "k8s.io/api/certificates/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + clientset "k8s.io/client-go/kubernetes" + fakeclient "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + tokenjws "k8s.io/cluster-bootstrap/token/jws" + + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" +) + +const ( + caCert = `-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTE5MTEyMDAwNDk0MloXDTI5MTExNzAwNDk0MlowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMqQ +ctECzA8yFSuVYupOUYgrTmfQeKe/9BaDWagaq7ow9+I2IvsfWFvlrD8QQr8sea6q +xjq7TV67Vb4RxBaoYDA+yI5vIcujWUxULun64lu3Q6iC1sj2UnmUpIdgazRXXEkZ +vxA6EbAnoxA0+lBOn1CZWl23IQ4s70o2hZ7wIp/vevB88RRRjqtvgc5elsjsbmDF +LS7L1Zuye8c6gS93bR+VjVmSIfr1IEq0748tIIyXjAVCWPVCvuP41MlfPc/JVpZD +uD2+pO6ZYREcdAnOf2eD4/eLOMKko4L1dSFy9JKM5PLnOC0Zk0AYOd1vS8DTAfxj +XPEIY8OBYFhlsxf4TE8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH/OYq8zyl1+zSTmuow3yI/15PL1 +dl8hB7IKnZNWmC/LTdm/+noh3Sb1IdRv6HkKg/GUn0UMuRUngLhju3EO4ozJPQcX +quaxzgmTKNWJ6ErDvRvWhGX0ZcbdBfZv+dowyRqzd5nlJ49hC+NrtFFQq6P05BYn +7SemguqeXmXwIj2Sa+1DeR6lRm9o8shAYjnyThUFqaMn18kI3SANJ5vk/3DFrPEO +CKC9EzFku2kuxg2dM12PbRGZQ2o0K6HEZgrrIKTPOy3ocb8r9M0aSFhjOV/NqGA4 +SaupXSW6XfvIi/UHoIbU3pNcsnUJGnQfQvip95XKk/gqcUr+m50vxgumxtA= +-----END CERTIFICATE-----` + tokenID = "123456" + tokenSecret = "abcdef1234567890" +) + +func CreateCertFakeClient(CaDir string) (clientset.Interface, error) { + // Create a fake client + client := fakeclient.NewSimpleClientset() + + // prepare cluster-info configmap + kubeconfig := buildSecureBootstrapKubeConfig("127.0.0.1", []byte(caCert), "somecluster") + kubeconfigBytes, err := clientcmd.Write(*kubeconfig) + if err != nil { + return client, err + } + + // Generate signature of the insecure kubeconfig + sig, err := tokenjws.ComputeDetachedSignature(string(kubeconfigBytes), tokenID, tokenSecret) + if err != nil { + return client, err + } + + cm := &fakeConfigMap{ + name: bootstrapapi.ConfigMapClusterInfo, + data: map[string]string{ + bootstrapapi.KubeConfigKey: string(kubeconfigBytes), + bootstrapapi.JWSSignatureKeyPrefix + tokenID: sig, + }, + } + + if err = cm.createOrUpdate(client); err != nil { + return client, err + } + + // prepare certificates + client = prepareCsrAndCert(client, CaDir) + + return client, nil +} + +type fakeConfigMap struct { + name string + data map[string]string +} + +func (c *fakeConfigMap) createOrUpdate(client clientset.Interface) error { + return createOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.name, + Namespace: metav1.NamespacePublic, + }, + Data: c.data, + }) +} + +// createOrUpdateConfigMap creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. +func createOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error { + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return errors.Wrap(err, "unable to create ConfigMap") + } + + if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(err, "unable to update ConfigMap") + } + } + return nil +} + +var ( + csr *certificatesv1.CertificateSigningRequest +) + +func prepareCsrAndCert(f *fakeclient.Clientset, testDir string) *fakeclient.Clientset { + f.PrependReactor("create", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + cAction, ok := action.(clienttesting.CreateAction) + if ok { + csr = cAction.GetObject().(*certificatesv1.CertificateSigningRequest) + } + return true, &certificatesv1.CertificateSigningRequest{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}, nil + default: + return false, nil, nil + } + }) + f.PrependReactor("list", "certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + switch action.GetResource().Version { + case "v1": + return true, &certificatesv1.CertificateSigningRequestList{Items: []certificatesv1.CertificateSigningRequest{{ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}}}}, nil + default: + return false, nil, nil + } + }) + f.PrependWatchReactor("certificatesigningrequests", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + switch action.GetResource().Version { + case "v1": + certData, err := signCertificate(csr, testDir) + if err != nil { + return true, nil, err + } + return true, &fakeWatch{ + version: action.GetResource().Version, + certificatePEM: certData, + }, nil + default: + return false, nil, nil + } + }) + return f +} + +type fakeWatch struct { + version string + certificatePEM []byte +} + +func (w *fakeWatch) Stop() { +} + +func (w *fakeWatch) ResultChan() <-chan watch.Event { + var csr runtime.Object + + switch w.version { + case "v1": + condition := certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + } + + csr = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{UID: "fake-uid"}, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + condition, + }, + Certificate: []byte(w.certificatePEM), + }, + } + } + + c := make(chan watch.Event, 1) + c <- watch.Event{ + Type: watch.Added, + Object: csr, + } + return c +} + +// buildSecureBootstrapKubeConfig makes a kubeconfig object that connects securely to the API Server for bootstrapping purposes (validating with the specified CA) +func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername string) *clientcmdapi.Config { + controlPlaneEndpoint := fmt.Sprintf("https://%s", endpoint) + bootstrapConfig := kubeconfigutil.CreateBasic(controlPlaneEndpoint, clustername, "token-bootstrap-client", caCert) + return bootstrapConfig +} + +func signCertificate(csr *certificatesv1.CertificateSigningRequest, testDir string) ([]byte, error) { + caFile := filepath.Join(testDir, "ca.crt") + caKeyFile := filepath.Join(testDir, "ca.key") + caLoader, err := dynamiccertificates.NewDynamicServingContentFromFiles("csr-controller", caFile, caKeyFile) + if err != nil { + return nil, fmt.Errorf("error reading CA cert file %q: %v", caFile, err) + } + + cap := &caProvider{ + caLoader: caLoader, + } + if err := cap.setCA(); err != nil { + return nil, err + } + + x509cr, err := parseCSR(csr.Spec.Request) + if err != nil { + return nil, fmt.Errorf("unable to parse csr %q: %v", csr.Name, err) + } + + currCA, err := cap.currentCA() + if err != nil { + return nil, err + } + + der, err := currCA.Sign(x509cr.Raw, PermissiveSigningPolicy{ + TTL: 365 * 24 * 3600 * time.Second, + Usages: csr.Spec.Usages, + Backdate: 5 * time.Minute, + Short: 8 * time.Hour, + Now: nil, + }) + if err != nil { + return nil, err + } + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), nil +} + +var serialNumberLimit = new(big.Int).Lsh(big.NewInt(1), 128) + +// CertificateAuthority implements a certificate authority that supports policy +// based signing. It's used by the signing controller. +type CertificateAuthority struct { + // RawCert is an optional field to determine if signing cert/key pairs have changed + RawCert []byte + // RawKey is an optional field to determine if signing cert/key pairs have changed + RawKey []byte + + Certificate *x509.Certificate + PrivateKey crypto.Signer +} + +// Sign signs a certificate request, applying a SigningPolicy and returns a DER +// encoded x509 certificate. +func (ca *CertificateAuthority) Sign(crDER []byte, policy SigningPolicy) ([]byte, error) { + cr, err := x509.ParseCertificateRequest(crDER) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate request: %v", err) + } + if err := cr.CheckSignature(); err != nil { + return nil, fmt.Errorf("unable to verify certificate request signature: %v", err) + } + + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, fmt.Errorf("unable to generate a serial number for %s: %v", cr.Subject.CommonName, err) + } + + tmpl := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: cr.Subject, + DNSNames: cr.DNSNames, + IPAddresses: cr.IPAddresses, + EmailAddresses: cr.EmailAddresses, + URIs: cr.URIs, + PublicKeyAlgorithm: cr.PublicKeyAlgorithm, + PublicKey: cr.PublicKey, + Extensions: cr.Extensions, + ExtraExtensions: cr.ExtraExtensions, + } + if err := policy.apply(tmpl, ca.Certificate.NotAfter); err != nil { + return nil, err + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Certificate, cr.PublicKey, ca.PrivateKey) + if err != nil { + return nil, fmt.Errorf("failed to sign certificate: %v", err) + } + return der, nil +} + +type caProvider struct { + caValue atomic.Value + caLoader *dynamiccertificates.DynamicCertKeyPairContent +} + +// setCA unconditionally stores the current cert/key content +func (p *caProvider) setCA() error { + certPEM, keyPEM := p.caLoader.CurrentCertKeyContent() + + certs, err := cert.ParseCertsPEM(certPEM) + if err != nil { + return fmt.Errorf("error reading CA cert file %q: %v", p.caLoader.Name(), err) + } + if len(certs) != 1 { + return fmt.Errorf("error reading CA cert file %q: expected 1 certificate, found %d", p.caLoader.Name(), len(certs)) + } + + key, err := keyutil.ParsePrivateKeyPEM(keyPEM) + if err != nil { + return fmt.Errorf("error reading CA key file %q: %v", p.caLoader.Name(), err) + } + priv, ok := key.(crypto.Signer) + if !ok { + return fmt.Errorf("error reading CA key file %q: key did not implement crypto.Signer", p.caLoader.Name()) + } + + ca := &CertificateAuthority{ + RawCert: certPEM, + RawKey: keyPEM, + + Certificate: certs[0], + PrivateKey: priv, + } + p.caValue.Store(ca) + + return nil +} + +// currentCA provides the current value of the CA. +// It always check for a stale value. This is cheap because it's all an in memory cache of small slices. +func (p *caProvider) currentCA() (*CertificateAuthority, error) { + certPEM, keyPEM := p.caLoader.CurrentCertKeyContent() + currCA := p.caValue.Load().(*CertificateAuthority) + if bytes.Equal(currCA.RawCert, certPEM) && bytes.Equal(currCA.RawKey, keyPEM) { + return currCA, nil + } + + // the bytes weren't equal, so we have to set and then load + if err := p.setCA(); err != nil { + return currCA, err + } + return p.caValue.Load().(*CertificateAuthority), nil +} + +// ParseCSR extracts the CSR from the bytes and decodes it. +func parseCSR(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE REQUEST" { + return nil, errors.New("PEM block type must be CERTIFICATE REQUEST") + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, err + } + return csr, nil +} + +// SigningPolicy validates a CertificateRequest before it's signed by the +// CertificateAuthority. It may default or otherwise mutate a certificate +// template. +type SigningPolicy interface { + // not-exporting apply forces signing policy implementations to be internal + // to this package. + apply(template *x509.Certificate, signerNotAfter time.Time) error +} + +// PermissiveSigningPolicy is the signing policy historically used by the local +// signer. +// +// - It forwards all SANs from the original signing request. +// - It sets allowed usages as configured in the policy. +// - It zeros all extensions. +// - It sets BasicConstraints to true. +// - It sets IsCA to false. +// - It validates that the signer has not expired. +// - It sets NotBefore and NotAfter: +// All certificates set NotBefore = Now() - Backdate. +// Long-lived certificates set NotAfter = Now() + TTL - Backdate. +// Short-lived certificates set NotAfter = Now() + TTL. +// All certificates truncate NotAfter to the expiration date of the signer. +type PermissiveSigningPolicy struct { + // TTL is used in certificate NotAfter calculation as described above. + TTL time.Duration + + // Usages are the allowed usages of a certificate. + Usages []certificatesv1.KeyUsage + + // Backdate is used in certificate NotBefore calculation as described above. + Backdate time.Duration + + // Short is the duration used to determine if the lifetime of a certificate should be considered short. + Short time.Duration + + // Now defaults to time.Now but can be stubbed for testing + Now func() time.Time +} + +func (p PermissiveSigningPolicy) apply(tmpl *x509.Certificate, signerNotAfter time.Time) error { + var now time.Time + if p.Now != nil { + now = p.Now() + } else { + now = time.Now() + } + + ttl := p.TTL + + usage, extUsages, err := keyUsagesFromStrings(p.Usages) + if err != nil { + return err + } + tmpl.KeyUsage = usage + tmpl.ExtKeyUsage = extUsages + + tmpl.ExtraExtensions = nil + tmpl.Extensions = nil + tmpl.BasicConstraintsValid = true + tmpl.IsCA = false + + tmpl.NotBefore = now.Add(-p.Backdate) + + if ttl < p.Short { + // do not backdate the end time if we consider this to be a short lived certificate + tmpl.NotAfter = now.Add(ttl) + } else { + tmpl.NotAfter = now.Add(ttl - p.Backdate) + } + + if !tmpl.NotAfter.Before(signerNotAfter) { + tmpl.NotAfter = signerNotAfter + } + + if !tmpl.NotBefore.Before(signerNotAfter) { + return fmt.Errorf("the signer has expired: NotAfter=%v", signerNotAfter) + } + + if !now.Before(signerNotAfter) { + return fmt.Errorf("refusing to sign a certificate that expired in the past: NotAfter=%v", signerNotAfter) + } + + return nil +} + +var keyUsageDict = map[certificatesv1.KeyUsage]x509.KeyUsage{ + certificatesv1.UsageSigning: x509.KeyUsageDigitalSignature, + certificatesv1.UsageDigitalSignature: x509.KeyUsageDigitalSignature, + certificatesv1.UsageContentCommitment: x509.KeyUsageContentCommitment, + certificatesv1.UsageKeyEncipherment: x509.KeyUsageKeyEncipherment, + certificatesv1.UsageKeyAgreement: x509.KeyUsageKeyAgreement, + certificatesv1.UsageDataEncipherment: x509.KeyUsageDataEncipherment, + certificatesv1.UsageCertSign: x509.KeyUsageCertSign, + certificatesv1.UsageCRLSign: x509.KeyUsageCRLSign, + certificatesv1.UsageEncipherOnly: x509.KeyUsageEncipherOnly, + certificatesv1.UsageDecipherOnly: x509.KeyUsageDecipherOnly, +} + +var extKeyUsageDict = map[certificatesv1.KeyUsage]x509.ExtKeyUsage{ + certificatesv1.UsageAny: x509.ExtKeyUsageAny, + certificatesv1.UsageServerAuth: x509.ExtKeyUsageServerAuth, + certificatesv1.UsageClientAuth: x509.ExtKeyUsageClientAuth, + certificatesv1.UsageCodeSigning: x509.ExtKeyUsageCodeSigning, + certificatesv1.UsageEmailProtection: x509.ExtKeyUsageEmailProtection, + certificatesv1.UsageSMIME: x509.ExtKeyUsageEmailProtection, + certificatesv1.UsageIPsecEndSystem: x509.ExtKeyUsageIPSECEndSystem, + certificatesv1.UsageIPsecTunnel: x509.ExtKeyUsageIPSECTunnel, + certificatesv1.UsageIPsecUser: x509.ExtKeyUsageIPSECUser, + certificatesv1.UsageTimestamping: x509.ExtKeyUsageTimeStamping, + certificatesv1.UsageOCSPSigning: x509.ExtKeyUsageOCSPSigning, + certificatesv1.UsageMicrosoftSGC: x509.ExtKeyUsageMicrosoftServerGatedCrypto, + certificatesv1.UsageNetscapeSGC: x509.ExtKeyUsageNetscapeServerGatedCrypto, +} + +// keyUsagesFromStrings will translate a slice of usage strings from the +// certificates API ("pkg/apis/certificates".KeyUsage) to x509.KeyUsage and +// x509.ExtKeyUsage types. +func keyUsagesFromStrings(usages []certificatesv1.KeyUsage) (x509.KeyUsage, []x509.ExtKeyUsage, error) { + var keyUsage x509.KeyUsage + var unrecognized []certificatesv1.KeyUsage + extKeyUsages := make(map[x509.ExtKeyUsage]struct{}) + for _, usage := range usages { + if val, ok := keyUsageDict[usage]; ok { + keyUsage |= val + } else if val, ok := extKeyUsageDict[usage]; ok { + extKeyUsages[val] = struct{}{} + } else { + unrecognized = append(unrecognized, usage) + } + } + + var sorted sortedExtKeyUsage + for eku := range extKeyUsages { + sorted = append(sorted, eku) + } + sort.Sort(sorted) + + if len(unrecognized) > 0 { + return 0, nil, fmt.Errorf("unrecognized usage values: %q", unrecognized) + } + + return keyUsage, sorted, nil +} + +type sortedExtKeyUsage []x509.ExtKeyUsage + +func (s sortedExtKeyUsage) Len() int { + return len(s) +} + +func (s sortedExtKeyUsage) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedExtKeyUsage) Less(i, j int) bool { + return s[i] < s[j] +} diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go new file mode 100644 index 00000000000..18c008bf6e9 --- /dev/null +++ b/pkg/yurthub/certificate/token/token.go @@ -0,0 +1,447 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "crypto/tls" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + certificatesv1 "k8s.io/api/certificates/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/user" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/certificate" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/projectinfo" + yurtutil "github.com/openyurtio/openyurt/pkg/util" + certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" + "github.com/openyurtio/openyurt/pkg/util/certmanager/store" + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" + "github.com/openyurtio/openyurt/pkg/util/token" + hubCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + YurtHubCSROrg = "openyurt:yurthub" + DefaultRootDir = "/var/lib" + hubPkiDirName = "pki" + hubCaFileName = "ca.crt" + bootstrapConfigFileName = "bootstrap-hub.conf" +) + +var ( + hubConfigFileName = fmt.Sprintf("%s.conf", projectinfo.GetHubName()) + serverCertNotReadyError = errors.New("hub server certificate") + apiServerClientCertNotReadyError = errors.New("APIServer client certificate") + caCertIsNotReadyError = errors.New("ca.crt file") +) + +type yurtHubCertManager struct { + client clientset.Interface + remoteServers []*url.URL + caCertHashes []string + apiServerClientCertManager certificate.Manager + hubServerCertManager certificate.Manager + apiServerClientCertStore certificate.FileStore + serverCertStore certificate.FileStore + hubRunDir string + hubName string + joinToken string + dialer *util.Dialer +} + +// NewYurtHubCertManager new a YurtCertificateManager instance +func NewYurtHubCertManager(client clientset.Interface, cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) (hubCert.YurtCertificateManager, error) { + var err error + + hubRunDir := cfg.RootDir + if len(cfg.RootDir) == 0 { + hubRunDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) + } + + ycm := &yurtHubCertManager{ + client: client, + remoteServers: cfg.RemoteServers, + hubRunDir: hubRunDir, + hubName: projectinfo.GetHubName(), + joinToken: cfg.JoinToken, + caCertHashes: cfg.CaCertHashes, + dialer: util.NewDialer("hub certificate manager"), + } + + // 1. verify that need to clean up stale certificates or not based on server addresses. + ycm.verifyServerAddrOrCleanup(cfg.RemoteServers) + + // 2. prepare client certificate manager for connecting remote kube-apiserver by yurthub. + ycm.apiServerClientCertStore, err = store.NewFileStoreWrapper(ycm.hubName, ycm.getPkiDir(), ycm.getPkiDir(), "", "") + if err != nil { + return ycm, errors.Wrap(err, "couldn't new client cert store") + } + ycm.apiServerClientCertManager, err = ycm.newAPIServerClientCertificateManager(ycm.apiServerClientCertStore, cfg.NodeName, cfg.YurtHubCertOrganizations) + if err != nil { + return ycm, errors.Wrap(err, "couldn't new apiserver client certificate manager") + } + + // 3. prepare yurthub server certificate manager + ycm.serverCertStore, err = store.NewFileStoreWrapper(fmt.Sprintf("%s-server", ycm.hubName), ycm.getPkiDir(), ycm.getPkiDir(), "", "") + if err != nil { + return ycm, errors.Wrap(err, "couldn't new hub server cert store") + } + + ycm.hubServerCertManager, err = ycm.newHubServerCertificateManager(ycm.serverCertStore, cfg.NodeName, cfg.CertIPs) + if err != nil { + return ycm, errors.Wrap(err, "couldn't new hub server certificate manager") + } + + return ycm, nil +} + +func removeDirContents(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, d := range files { + err = os.RemoveAll(filepath.Join(dir, d.Name())) + if err != nil { + return err + } + } + return nil +} + +func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { + if cfg, err := clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err == nil { + cluster := kubeconfigutil.GetClusterFromKubeConfig(cfg) + if serverURL, err := url.Parse(cluster.Server); err != nil { + klog.Errorf("couldn't get server info from %s, %v", ycm.getBootstrapConfFile(), err) + } else { + for i := range servers { + if servers[i].Host == serverURL.Host { + klog.Infof("apiServer name %s not changed", cluster.Server) + return + } + } + } + + klog.Infof("config for apiServer %s found, need to recycle for new server %v", cluster.Server, servers) + removeDirContents(ycm.hubRunDir) + } +} + +// Start init certificate manager and certs for hub agent +func (ycm *yurtHubCertManager) Start() { + err := ycm.prepareConfigAndCaFile() + if err != nil { + klog.Errorf("failed to prepare config and ca file, %v", err) + return + } + + ycm.apiServerClientCertManager.Start() + ycm.hubServerCertManager.Start() +} + +// prepareConfigAndCaFile is used to create the following three files. +// - /var/lib/yurthub/bootstrap-hub.conf +// - /var/lib/yurthub/yurthub.conf +// - /var/lib/yurthub/pki/ca.crt +// if these files already exist, just reuse them. +func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { + var tlsBootstrapCfg *clientcmdapi.Config + + // 1. prepare bootstrap config file(/var/lib/yurthub/bootstrap-hub.conf) for yurthub + if exist, err := util.FileExists(ycm.getBootstrapConfFile()); err != nil { + return errors.Wrap(err, "couldn't stat bootstrap config file") + } else if !exist { + if tlsBootstrapCfg, err = ycm.retrieveHubBootstrapConfig(ycm.joinToken); err != nil { + return errors.Wrap(err, "failed to retrieve bootstrap config") + } + } else { + klog.V(2).Infof("%s file already exists, so reuse it", ycm.getBootstrapConfFile()) + if tlsBootstrapCfg, err = clientcmd.LoadFromFile(ycm.getBootstrapConfFile()); err != nil { + return errors.Wrap(err, "couldn't load bootstrap config file") + } + } + + // 2. prepare kubeconfig file(/var/lib/yurthub/yurthub.conf) for yurthub + if exist, err := util.FileExists(ycm.GetHubConfFile()); err != nil { + return errors.Wrap(err, "couldn't stat hub kubeconfig file") + } else if !exist { + hubCfg := createHubConfig(tlsBootstrapCfg, ycm.apiServerClientCertStore.CurrentPath()) + if err = kubeconfigutil.WriteToDisk(ycm.GetHubConfFile(), hubCfg); err != nil { + return errors.Wrapf(err, "couldn't save %s to disk", hubConfigFileName) + } + } else { + klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetHubConfFile()) + } + + // 3. prepare ca.crt file(/var/lib/yurthub/pki/ca.crt) for yurthub + if exist, err := util.FileExists(ycm.GetCaFile()); err != nil { + return errors.Wrap(err, "couldn't stat ca.crt file") + } else if !exist { + cluster := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) + if cluster != nil { + if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil { + return errors.Wrap(err, "couldn't save the CA certificate to disk") + } + } else { + return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile()) + } + } else { + klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile()) + } + + return nil +} + +// Stop the cert manager loop +func (ycm *yurtHubCertManager) Stop() { + ycm.apiServerClientCertManager.Stop() + ycm.hubServerCertManager.Stop() +} + +// Ready is used for checking client/server/ca certificates are prepared completely or not. +func (ycm *yurtHubCertManager) Ready() bool { + var errs []error + if ycm.GetHubServerCert() == nil { + errs = append(errs, serverCertNotReadyError) + } + + if ycm.GetAPIServerClientCert() == nil { + errs = append(errs, apiServerClientCertNotReadyError) + } + + if exist, _ := util.FileExists(ycm.GetCaFile()); !exist { + errs = append(errs, caCertIsNotReadyError) + } + + if len(errs) != 0 { + klog.Errorf("hub certificates are not ready: %s", utilerrors.NewAggregate(errs).Error()) + return false + } + return true +} + +// UpdateBootstrapConf is used for revising bootstrap conf file by new bearer token. +func (ycm *yurtHubCertManager) UpdateBootstrapConf(joinToken string) error { + _, err := ycm.retrieveHubBootstrapConfig(joinToken) + return err +} + +// getPkiDir returns the directory for storing hub agent pki +func (ycm *yurtHubCertManager) getPkiDir() string { + return filepath.Join(ycm.hubRunDir, hubPkiDirName) +} + +// getBootstrapConfFile returns the path of yurthub bootstrap conf file +func (ycm *yurtHubCertManager) getBootstrapConfFile() string { + return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName) +} + +// GetCaFile returns the path of ca file +func (ycm *yurtHubCertManager) GetCaFile() string { + return filepath.Join(ycm.getPkiDir(), hubCaFileName) +} + +// GetHubConfFile returns the path of yurtHub config file path +func (ycm *yurtHubCertManager) GetHubConfFile() string { + return filepath.Join(ycm.hubRunDir, hubConfigFileName) +} + +func (ycm *yurtHubCertManager) GetAPIServerClientCert() *tls.Certificate { + return ycm.apiServerClientCertManager.Current() +} + +func (ycm *yurtHubCertManager) GetHubServerCert() *tls.Certificate { + return ycm.hubServerCertManager.Current() +} + +func (ycm *yurtHubCertManager) GetHubServerCertFile() string { + return ycm.serverCertStore.CurrentPath() +} + +// newAPIServerClientCertificateManager create a certificate manager for yurthub component to prepare client certificate +// that used to proxy requests to remote kube-apiserver. +func (ycm *yurtHubCertManager) newAPIServerClientCertificateManager(fileStore certificate.FileStore, nodeName string, hubCertOrganizations []string) (certificate.Manager, error) { + orgs := []string{YurtHubCSROrg, user.NodesGroup} + for _, v := range hubCertOrganizations { + if v != YurtHubCSROrg && v != user.NodesGroup { + orgs = append(orgs, v) + } + } + + return certfactory.NewCertManagerFactoryWithFnAndStore(ycm.generateCertClientFn, fileStore).New(&certfactory.CertManagerConfig{ + ComponentName: ycm.hubName, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organizations: orgs, + SignerName: certificatesv1.KubeAPIServerClientSignerName, + }) +} + +func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { + var kubeconfig *restclient.Config + var err error + if !yurtutil.IsNil(ycm.client) { + return ycm.client, nil + } + + // If we have a valid certificate, use that to fetch CSRs, Otherwise use the bootstrap conf file. + if current != nil { + klog.V(2).Infof("use %s config to create csr client", ycm.hubName) + kubeconfig, err = clientcmd.BuildConfigFromFlags("", ycm.GetHubConfFile()) + if err != nil { + klog.Errorf("could not load %s kube config(%s), %v", ycm.hubName, ycm.GetHubConfFile(), err) + return nil, errors.Wrap(err, "could not load hub kubeconfig file") + } + } else { + klog.V(2).Infof("use bootstrap client config to create csr client") + kubeconfig, err = clientcmd.BuildConfigFromFlags("", ycm.getBootstrapConfFile()) + if err != nil { + klog.Errorf("could not load bootstrap config in clientFn, %v", err) + return nil, errors.Wrap(err, "couldn't load hub bootstrap file") + } + } + + if kubeconfig == nil { + return nil, errors.New("kubeconfig for client certificate is not ready") + } + kubeconfig.Host = findActiveRemoteServer(ycm.remoteServers).String() + // re-fix dial for conn management + kubeconfig.Dial = ycm.dialer.DialContext + + // avoid tcp conn leak: certificate rotated, so close old tcp conn that used to rotate certificate + klog.V(2).Infof("avoid tcp conn leak, close old tcp conn that used to rotate certificate") + ycm.dialer.Close(strings.Trim(kubeconfig.Host, "https://")) + + return clientset.NewForConfig(kubeconfig) +} + +// newHubServerCertificateManager create a certificate manager for yurthub component to prepare server certificate +// that used to handle requests from clients on edge nodes. +func (ycm *yurtHubCertManager) newHubServerCertificateManager(fileStore certificate.FileStore, nodeName string, certIPs []net.IP) (certificate.Manager, error) { + kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { + // waiting for the certificate is generated + _ = wait.PollInfinite(5*time.Second, func() (bool, error) { + // keep polling until the yurthub client certificate is signed + if ycm.apiServerClientCertManager.Current() != nil { + return true, nil + } + klog.Infof("waiting for the controller-manager to sign the %s client certificate", ycm.hubName) + return false, nil + }) + + if !yurtutil.IsNil(ycm.client) { + return ycm.client, nil + } + + return kubeconfigutil.ClientSetFromFile(ycm.GetHubConfFile()) + } + + // create a certificate manager for the yurthub server and run the csr approver for both yurthub + return certfactory.NewCertManagerFactoryWithFnAndStore(kubeClientFn, fileStore).New(&certfactory.CertManagerConfig{ + ComponentName: fmt.Sprintf("%s-server", ycm.hubName), + SignerName: certificatesv1.KubeletServingSignerName, + ForServerUsage: true, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organizations: []string{user.NodesGroup}, + IPs: certIPs, + }) +} + +func (ycm *yurtHubCertManager) retrieveHubBootstrapConfig(joinToken string) (*clientcmdapi.Config, error) { + // retrieve bootstrap config info from cluster-info configmap by bootstrap token + serverAddr := findActiveRemoteServer(ycm.remoteServers).Host + if cfg, err := token.RetrieveValidatedConfigInfo(ycm.client, &token.BootstrapData{ + ServerAddr: serverAddr, + JoinToken: joinToken, + CaCertHashes: ycm.caCertHashes, + }); err != nil { + return nil, errors.Wrap(err, "couldn't retrieve bootstrap config info") + } else { + clusterInfo := kubeconfigutil.GetClusterFromKubeConfig(cfg) + tlsBootstrapCfg := kubeconfigutil.CreateWithToken( + fmt.Sprintf("https://%s", serverAddr), + "kubernetes", + "token-bootstrap-client", + clusterInfo.CertificateAuthorityData, + joinToken, + ) + if err = kubeconfigutil.WriteToDisk(ycm.getBootstrapConfFile(), tlsBootstrapCfg); err != nil { + return nil, errors.Wrap(err, "couldn't save bootstrap-hub.conf to disk") + } + + return tlsBootstrapCfg, nil + } +} + +func createHubConfig(tlsBootstrapCfg *clientcmdapi.Config, pemPath string) *clientcmdapi.Config { + cluster := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) + + // Build resulting kubeconfig. + return &clientcmdapi.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { + Server: cluster.Server, + CertificateAuthority: cluster.CertificateAuthority, + CertificateAuthorityData: cluster.CertificateAuthorityData, + }}, + // Define auth based on the obtained client cert. + AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": { + ClientCertificate: pemPath, + ClientKey: pemPath, + }}, + // Define a context that connects the auth info and cluster, and set it as the default + Contexts: map[string]*clientcmdapi.Context{"default-context": { + Cluster: "default-cluster", + AuthInfo: "default-auth", + Namespace: "default", + }}, + CurrentContext: "default-context", + } +} + +func findActiveRemoteServer(servers []*url.URL) *url.URL { + if len(servers) == 0 { + return nil + } else if len(servers) == 1 { + return servers[0] + } + + for i := range servers { + _, err := net.DialTimeout("tcp", servers[i].Host, 5*time.Second) + if err == nil { + return servers[i] + } + } + + return servers[0] +} diff --git a/pkg/yurthub/certificate/token/token_test.go b/pkg/yurthub/certificate/token/token_test.go new file mode 100644 index 00000000000..bb9895821b7 --- /dev/null +++ b/pkg/yurthub/certificate/token/token_test.go @@ -0,0 +1,314 @@ +/* +Copyright 2020 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "fmt" + "io/ioutil" + "net" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" +) + +func Test_removeDirContents(t *testing.T) { + dir, err := ioutil.TempDir("", "yurthub-test-removeDirContents") + if err != nil { + t.Fatalf("Unable to create the test directory %q: %v", dir, err) + } + defer func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("Unable to clean up test directory %q: %v", dir, err) + } + }() + + tempFile := dir + "/tmp.txt" + if err = ioutil.WriteFile(tempFile, nil, 0600); err != nil { + t.Fatalf("Unable to create the test file %q: %v", tempFile, err) + } + + type args struct { + dir string + } + tests := []struct { + name string + args args + file string + wantErr bool + }{ + {"no input dir", args{}, "", true}, + {"input dir exist", args{dir}, tempFile, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err = removeDirContents(tt.args.dir); (err != nil) != tt.wantErr { + t.Errorf("removeDirContents() error = %v, wantErr %v", err, tt.wantErr) + } + + if tt.file != "" { + if _, err = os.Stat(tt.file); err == nil || !os.IsNotExist(err) { + t.Errorf("after remote dir content, no file should exist. file:%s", tt.file) + } + } + }) + } +} + +func TestGetHubConfFile(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + stopCh := make(chan struct{}) + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: "", + path: filepath.Join("/var/lib", projectinfo.GetHubName(), fmt.Sprintf("%s.conf", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp", + path: filepath.Join("/tmp", fmt.Sprintf("%s.conf", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + cfg := &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: tc.rootDir, + } + + mgr, err := NewYurtHubCertManager(nil, cfg, stopCh) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubConfFile() != tc.path { + t.Errorf("expect hub conf file %s, but got %s", tc.path, mgr.GetHubConfFile()) + } + }) + } +} + +func TestGetCaFile(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + stopCh := make(chan struct{}) + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: "", + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", "ca.crt"), + }, + "define root dir": { + rootDir: "/tmp", + path: filepath.Join("/tmp", "pki", "ca.crt"), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + cfg := &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: tc.rootDir, + } + + mgr, err := NewYurtHubCertManager(nil, cfg, stopCh) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetCaFile() != tc.path { + t.Errorf("expect ca.crt file %s, but got %s", tc.path, mgr.GetCaFile()) + } + }) + } +} + +func TestGetHubServerCertFile(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + stopCh := make(chan struct{}) + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: "", + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp", + path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + cfg := &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: tc.rootDir, + } + + mgr, err := NewYurtHubCertManager(nil, cfg, stopCh) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubServerCertFile() != tc.path { + t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) + } + }) + } +} + +var ( + joinToken = "123456.abcdef1234567890" + rootDir = "/tmp/token/cert" +) + +func TestUpdateBootstrapConf(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + stopCh := make(chan struct{}) + testcases := map[string]struct { + joinToken string + err error + }{ + "normal update": { + joinToken: joinToken, + err: nil, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + client, err := testdata.CreateCertFakeClient("./testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return + } + + mgr, err := NewYurtHubCertManager(client, &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: rootDir, + JoinToken: tc.joinToken, + }, stopCh) + if err != nil { + t.Errorf("failed to new yurt cert manager, %v", err) + return + } + + resultErr := mgr.UpdateBootstrapConf(tc.joinToken) + if resultErr != tc.err { + t.Errorf("expect err is %v, but got %v", tc.err, resultErr) + } + mgr.Stop() + }) + } + os.RemoveAll(rootDir) +} + +func TestReady(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + stopCh := make(chan struct{}) + + client, err := testdata.CreateCertFakeClient("./testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return + } + + mgr, err := NewYurtHubCertManager(client, &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + }, stopCh) + if err != nil { + t.Errorf("failed to new yurt cert manager, %v", err) + return + } + mgr.Start() + + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if mgr.Ready() { + return true, nil + } + return false, nil + }) + + if err != nil { + t.Errorf("certificates are not ready, %v", err) + } + + mgr.Stop() + + // reuse the config and ca file + t.Logf("go to check the reuse of config and ca file") + newMgr, err := NewYurtHubCertManager(client, &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + }, stopCh) + if err != nil { + t.Errorf("failed to new another yurt cert manager, %v", err) + return + } + newMgr.Start() + if !newMgr.Ready() { + t.Errorf("certificates can not be reused") + } + newMgr.Stop() + + os.RemoveAll(rootDir) +} diff --git a/pkg/yurthub/healthchecker/fake_checker.go b/pkg/yurthub/healthchecker/fake_checker.go index 84a166833a8..fb9a6ce81d0 100644 --- a/pkg/yurthub/healthchecker/fake_checker.go +++ b/pkg/yurthub/healthchecker/fake_checker.go @@ -52,6 +52,16 @@ func (fc *fakeChecker) RenewKubeletLeaseTime() { return } +func (fc *fakeChecker) PickHealthyServer() (*url.URL, error) { + for server := range fc.settings { + if fc.healthy { + return url.Parse(server) + } + } + + return nil, nil +} + // NewFakeChecker creates a fake checker func NewFakeChecker(healthy bool, settings map[string]int) MultipleBackendsHealthChecker { return &fakeChecker{ diff --git a/pkg/yurthub/healthchecker/health_checker.go b/pkg/yurthub/healthchecker/health_checker.go index 8b7e32e54eb..eb78ac7e8f9 100644 --- a/pkg/yurthub/healthchecker/health_checker.go +++ b/pkg/yurthub/healthchecker/health_checker.go @@ -168,6 +168,16 @@ func (hc *cloudAPIServerHealthChecker) IsHealthy() bool { return false } +func (hc *cloudAPIServerHealthChecker) PickHealthyServer() (*url.URL, error) { + for server, prober := range hc.probers { + if prober.IsHealthy() { + return url.Parse(server) + } + } + + return nil, nil +} + // BackendHealthyStatus returns the healthy stats of specified server func (hc *cloudAPIServerHealthChecker) BackendHealthyStatus(server *url.URL) bool { if prober, ok := hc.probers[server.String()]; ok { diff --git a/pkg/yurthub/healthchecker/interfaces.go b/pkg/yurthub/healthchecker/interfaces.go index 18a9ca99db2..b74ce689eb3 100644 --- a/pkg/yurthub/healthchecker/interfaces.go +++ b/pkg/yurthub/healthchecker/interfaces.go @@ -35,6 +35,7 @@ type HealthChecker interface { type MultipleBackendsHealthChecker interface { HealthChecker BackendHealthyStatus(server *url.URL) bool + PickHealthyServer() (*url.URL, error) } // BackendProber is used to send heartbeat to backend and verify backend diff --git a/pkg/yurthub/kubernetes/rest/config.go b/pkg/yurthub/kubernetes/rest/config.go index d4b5fe01aa4..1b1a5010058 100644 --- a/pkg/yurthub/kubernetes/rest/config.go +++ b/pkg/yurthub/kubernetes/rest/config.go @@ -20,78 +20,48 @@ import ( "net/url" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" - "github.com/openyurtio/openyurt/pkg/yurthub/util" ) type RestConfigManager struct { - remoteServers []*url.URL - checker healthchecker.MultipleBackendsHealthChecker - certManager interfaces.YurtCertificateManager + checker healthchecker.MultipleBackendsHealthChecker + certManager certificate.YurtCertificateManager } // NewRestConfigManager creates a *RestConfigManager object -func NewRestConfigManager(cfg *config.YurtHubConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.MultipleBackendsHealthChecker) (*RestConfigManager, error) { +func NewRestConfigManager(certManager certificate.YurtCertificateManager, healthChecker healthchecker.MultipleBackendsHealthChecker) (*RestConfigManager, error) { mgr := &RestConfigManager{ - remoteServers: cfg.RemoteServers, - checker: healthChecker, - certManager: certMgr, + checker: healthChecker, + certManager: certManager, } return mgr, nil } // GetRestConfig gets rest client config according to the mode of certificateManager func (rcm *RestConfigManager) GetRestConfig(needHealthyServer bool) *rest.Config { - return rcm.getHubselfRestConfig(needHealthyServer) -} - -// getHubselfRestConfig gets rest client config from hub agent conf file. -func (rcm *RestConfigManager) getHubselfRestConfig(needHealthyServer bool) *rest.Config { - healthyServer := rcm.remoteServers[0] + var healthyServer *url.URL if needHealthyServer { - healthyServer = rcm.getHealthyServer() + healthyServer, _ = rcm.checker.PickHealthyServer() if healthyServer == nil { klog.Infof("all of remote servers are unhealthy, so return nil for rest config") return nil } } - // certificate expired, rest config can not be used to connect remote server, - // so return nil for rest config - if rcm.certManager.Current() == nil { - klog.Infof("certificate expired, so return nil for rest config") + kubeconfig, err := clientcmd.BuildConfigFromFlags("", rcm.certManager.GetHubConfFile()) + if err != nil { + klog.Errorf("could not load kube config(%s), %v", rcm.certManager.GetHubConfFile(), err) return nil } - hubConfFile := rcm.certManager.GetConfFilePath() - if isExist, _ := util.FileExists(hubConfFile); isExist { - cfg, err := util.LoadRESTClientConfig(hubConfFile) - if err != nil { - klog.Errorf("could not get rest config for %s, %v", hubConfFile, err) - return nil - } - + if healthyServer != nil { // re-fix host connecting healthy server - cfg.Host = healthyServer.String() - klog.Infof("re-fix hub rest config host successfully with server %s", cfg.Host) - return cfg - } - - klog.Errorf("%s config file(%s) is not exist", projectinfo.GetHubName(), hubConfFile) - return nil -} - -// getHealthyServer is used to get a healthy server -func (rcm *RestConfigManager) getHealthyServer() *url.URL { - for _, server := range rcm.remoteServers { - if rcm.checker.BackendHealthyStatus(server) { - return server - } + kubeconfig.Host = healthyServer.String() + klog.Infof("re-fix hub rest config host successfully with server %s", kubeconfig.Host) } - return nil + return kubeconfig } diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go index e54fad377d6..2affa94effc 100644 --- a/pkg/yurthub/kubernetes/rest/config_test.go +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -17,142 +17,90 @@ limitations under the License. package rest import ( - "bytes" + "net" "net/url" "os" - "path/filepath" "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" - "github.com/openyurtio/openyurt/pkg/yurthub/util/fs" ) var ( - certificatePEM = []byte(`-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 -MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV -tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV -HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv -0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV -NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl -nkVA6wyOSDYBf3o= ------END CERTIFICATE-----`) - keyPEM = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC -PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M -zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf -G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg -XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK -iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar -e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX -54LzHNk/+Q== ------END RSA PRIVATE KEY-----`) - yurthubCon = `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: temp - server: https://10.10.10.113:6443 - name: default-cluster -contexts: -- context: - cluster: default-cluster - namespace: default - user: default-auth - name: default-context -current-context: default-context -kind: Config -preferences: {} -users: -- name: default-auth - user: - client-certificate: /tmp/pki/yurthub-current.pem - client-key: /tmp/pki/yurthub-current.pem -` - testDir = "/tmp/pki/" + testDir = "/tmp/rest/" ) -var fsOperator fs.FileSystemOperator - func TestGetRestConfig(t *testing.T) { - os.RemoveAll(testDir) - var err error - remoteServers := map[string]int{"https://10.10.10.113:6443": 2} + stopCh := make(chan struct{}) + nodeName := "foo" + servers := map[string]int{"https://10.10.10.113:6443": 2} u, _ := url.Parse("https://10.10.10.113:6443") - fakeHealthchecker := healthchecker.NewFakeChecker(true, remoteServers) - defer func() { - if err := os.RemoveAll(testDir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", testDir, err) - } - }() - - // store the kubelet ca file - caFile := filepath.Join(testDir, "ca.crt") - if err := fsOperator.CreateFile(filepath.Join(testDir, "ca.crt"), certificatePEM); err != nil { - t.Fatalf("Unable to create the file %q: %v", caFile, err) + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + fakeHealthyChecker := healthchecker.NewFakeChecker(false, servers) + + client, err := testdata.CreateCertFakeClient("../../certificate/token/testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return } - - // store the kubelet-pair.pem file - pairFile := filepath.Join(testDir, "kubelet-pair.pem") - pd := bytes.Join([][]byte{certificatePEM, keyPEM}, []byte("\n")) - if err := fsOperator.CreateFile(filepath.Join(testDir, "kubelet-pair.pem"), pd); err != nil { - t.Fatalf("Unable to create the file %q: %v", pairFile, err) + certManager, err := token.NewYurtHubCertManager(client, &config.YurtHubConfiguration{ + NodeName: nodeName, + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: testDir, + JoinToken: "123456.abcdef1234567890", + }, stopCh) + if err != nil { + t.Errorf("failed to create certManager, %v", err) + return } + certManager.Start() + defer certManager.Stop() + defer os.RemoveAll(testDir) - // store the yurthub-current.pem - yurthubCurrent := filepath.Join(testDir, "yurthub-current.pem") - if err := fsOperator.CreateFile(filepath.Join(testDir, "yurthub-current.pem"), pd); err != nil { - t.Fatalf("Unable to create the file %q: %v", yurthubCurrent, err) - } + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if certManager.Ready() { + return true, nil + } + return false, nil + }) - // set the YurtHubConfiguration - cfg := &config.YurtHubConfiguration{ - RootDir: testDir, - RemoteServers: []*url.URL{u}, - KubeletRootCAFilePath: caFile, - KubeletPairFilePath: pairFile, + if err != nil { + t.Errorf("certificates are not ready, %v", err) } - tests := []struct { - desc string - mode string + rcm, _ := NewRestConfigManager(certManager, fakeHealthyChecker) + + testcases := map[string]struct { + needHealthyServer bool + cfgIsNil bool }{ - {desc: "hubself mode", mode: "hubself"}, + "do not need healthy server": { + needHealthyServer: false, + cfgIsNil: false, + }, + "need healthy server": { + needHealthyServer: true, + cfgIsNil: true, + }, } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - cfg.CertMgrMode = tt.mode - var certMgr interfaces.YurtCertificateManager - if tt.mode == "hubself" { - certMgr, err = hubself.NewFakeYurtHubCertManager(testDir, yurthubCon, string(certificatePEM), string(keyPEM)) - certMgr.Start() - } - - if err != nil { - t.Errorf("failed to create %s certManager: %v", tt.mode, err) - } - - rcm, err := NewRestConfigManager(cfg, certMgr, fakeHealthchecker) - if err != nil { - t.Errorf("failed to create RestConfigManager: %v", err) - } - rc := rcm.GetRestConfig(true) - if tt.mode == "hubself" { - if rc.Host != u.String() || rc.TLSClientConfig.CertFile != yurthubCurrent || rc.TLSClientConfig.KeyFile != yurthubCurrent { - t.Errorf("The information in rest.Config is not correct: %s", tt.mode) + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + cfg := rcm.GetRestConfig(tc.needHealthyServer) + if tc.cfgIsNil { + if cfg != nil { + t.Errorf("expect rest config is nil, but got %v", cfg) } } else { - if rc.Host != u.String() || rc.TLSClientConfig.CAFile != caFile || rc.TLSClientConfig.KeyFile != pairFile { - t.Errorf("The information in rest.Config is not correct: %s", tt.mode) + if cfg == nil { + t.Errorf("expect non nil rest config, but got nil") } } }) diff --git a/pkg/yurthub/otaupdate/ota_test.go b/pkg/yurthub/otaupdate/ota_test.go index 870e479907c..36936f5dc6c 100644 --- a/pkg/yurthub/otaupdate/ota_test.go +++ b/pkg/yurthub/otaupdate/ota_test.go @@ -19,7 +19,6 @@ package otaupdate import ( "net/http" "net/http/httptest" - "net/url" "testing" "github.com/gorilla/mux" @@ -28,7 +27,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" @@ -166,13 +164,9 @@ func TestUpdatePod(t *testing.T) { } func TestHealthyCheck(t *testing.T) { - u, _ := url.Parse("https://10.10.10.113:6443") fakeHealthchecker := healthchecker.NewFakeChecker(false, nil) - cfg := &config.YurtHubConfiguration{ - RemoteServers: []*url.URL{u}, - } - rcm, err := rest.NewRestConfigManager(cfg, nil, fakeHealthchecker) + rcm, err := rest.NewRestConfigManager(nil, fakeHealthchecker) if err != nil { t.Fatal(err) } diff --git a/pkg/yurthub/server/certificate.go b/pkg/yurthub/server/certificate.go index ff356b1304a..028d88a2225 100644 --- a/pkg/yurthub/server/certificate.go +++ b/pkg/yurthub/server/certificate.go @@ -21,8 +21,7 @@ import ( "fmt" "net/http" - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" ) const ( @@ -31,7 +30,7 @@ const ( // updateTokenHandler returns a http handler that update bootstrap token in the bootstrap-hub.conf file // in order to update node certificate when both node certificate and old join token expires -func updateTokenHandler(certificateMgr interfaces.YurtCertificateManager) http.Handler { +func updateTokenHandler(certificateMgr certificate.YurtCertificateManager) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { tokens := make(map[string]string) decoder := json.NewDecoder(r.Body) @@ -49,7 +48,7 @@ func updateTokenHandler(certificateMgr interfaces.YurtCertificateManager) http.H return } - err = certificateMgr.Update(&config.YurtHubConfiguration{JoinToken: joinToken}) + err = certificateMgr.UpdateBootstrapConf(joinToken) if err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "could not update bootstrap token, %v", err) diff --git a/pkg/yurthub/server/certificate_test.go b/pkg/yurthub/server/certificate_test.go index 59e2f9b5359..aca35250e41 100644 --- a/pkg/yurthub/server/certificate_test.go +++ b/pkg/yurthub/server/certificate_test.go @@ -19,15 +19,59 @@ package server import ( "bytes" "encoding/json" + "net" "net/http" "net/http/httptest" + "net/url" + "os" "testing" + "time" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" +) + +var ( + testDir = "/tmp/server/cert" ) func TestUpdateTokenHandler(t *testing.T) { - certMgr := &hubself.FakeYurtHubCertManager{} + stopCh := make(chan struct{}) + u, _ := url.Parse("https://10.10.10.113:6443") + remoteServers := []*url.URL{u} + certIPs := []net.IP{net.ParseIP("127.0.0.1")} + client, err := testdata.CreateCertFakeClient("../certificate/token/testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return + } + certManager, err := token.NewYurtHubCertManager(client, &config.YurtHubConfiguration{ + NodeName: "foo", + RemoteServers: remoteServers, + CertIPs: certIPs, + RootDir: testDir, + JoinToken: "123456.abcdef1234567890", + }, stopCh) + if err != nil { + t.Errorf("failed to create certManager, %v", err) + return + } + certManager.Start() + defer certManager.Stop() + defer os.RemoveAll(testDir) + + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if certManager.Ready() { + return true, nil + } + return false, nil + }) + if err != nil { + t.Errorf("certificates are not ready, %v", err) + } testcases := map[string]struct { body map[string]string @@ -39,7 +83,7 @@ func TestUpdateTokenHandler(t *testing.T) { }, "update join token normally": { body: map[string]string{ - tokenKey: "123456", + tokenKey: "123456.abcdef1234567890", }, statusCode: http.StatusOK, }, @@ -53,7 +97,7 @@ func TestUpdateTokenHandler(t *testing.T) { t.Fatal(err) } resp := httptest.NewRecorder() - updateTokenHandler(certMgr).ServeHTTP(resp, req) + updateTokenHandler(certManager).ServeHTTP(resp, req) if resp.Code != tt.statusCode { t.Errorf("expect status code %d, but got %d", tt.statusCode, resp.Code) diff --git a/pkg/yurthub/server/nonresource_test.go b/pkg/yurthub/server/nonresource_test.go index 5ceea6e0617..da61c6ebbaf 100644 --- a/pkg/yurthub/server/nonresource_test.go +++ b/pkg/yurthub/server/nonresource_test.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "net/http" "net/http/httptest" - "net/url" "os" "path/filepath" "strings" @@ -36,7 +35,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" fakerest "k8s.io/client-go/rest/fake" - "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" @@ -53,13 +51,10 @@ func TestLocalCacheHandler(t *testing.T) { } sw := cachemanager.NewStorageWrapper(dStorage) - u, _ := url.Parse("https://10.10.10.113:6443") + //u, _ := url.Parse("https://10.10.10.113:6443") fakeHealthChecker := healthchecker.NewFakeChecker(false, nil) - cfg := &config.YurtHubConfiguration{ - RemoteServers: []*url.URL{u}, - } - rcm, err := rest.NewRestConfigManager(cfg, nil, fakeHealthChecker) + rcm, err := rest.NewRestConfigManager(nil, fakeHealthChecker) if err != nil { t.Fatal(err) } diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 73f2c388a9a..fafe8b7c38c 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -21,22 +21,15 @@ import ( "fmt" "net" "net/http" - "time" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" - certificatesv1 "k8s.io/api/certificates/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" "github.com/openyurtio/openyurt/pkg/profile" - "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/util/certmanager" - certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" ota "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate" ) @@ -59,7 +52,7 @@ type yurtHubServer struct { // NewYurtHubServer creates a Server object func NewYurtHubServer(cfg *config.YurtHubConfiguration, - certificateMgr interfaces.YurtCertificateManager, + certificateMgr certificate.YurtCertificateManager, proxyHandler http.Handler, rest *rest.RestConfigManager) (Server, error) { hubMux := mux.NewRouter() @@ -152,8 +145,7 @@ func (s *yurtHubServer) Run() { } // registerHandler registers handlers for yurtHubServer, and yurtHubServer can handle requests like profiling, healthz, update token. -func registerHandlers(c *mux.Router, cfg *config.YurtHubConfiguration, certificateMgr interfaces.YurtCertificateManager, - rest *rest.RestConfigManager) { +func registerHandlers(c *mux.Router, cfg *config.YurtHubConfiguration, certificateMgr certificate.YurtCertificateManager, rest *rest.RestConfigManager) { // register handlers for update join token c.Handle("/v1/token", updateTokenHandler(certificateMgr)).Methods("POST", "PUT") @@ -181,56 +173,17 @@ func healthz(w http.ResponseWriter, _ *http.Request) { } // GenUseCertMgrAndTLSConfig create a certificate manager for the yurthub server and generate a TLS configuration -func GenUseCertMgrAndTLSConfig( - restConfigMgr *rest.RestConfigManager, - certificateMgr interfaces.YurtCertificateManager, - certDir, nodeName string, - certIPs []net.IP, - stopCh <-chan struct{}) (*tls.Config, error) { - cfg := restConfigMgr.GetRestConfig(false) - if cfg == nil { - return nil, fmt.Errorf("failed to prepare rest config based ong hub agent client certificate") - } - - clientSet, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - // create a certificate manager for the yurthub server and run the csr approver for both yurthub - serverCertMgr, err := certfactory.NewCertManagerFactory(clientSet).New(&certfactory.CertManagerConfig{ - CertDir: certDir, - ComponentName: fmt.Sprintf("%s-server", projectinfo.GetHubName()), - SignerName: certificatesv1.KubeletServingSignerName, - ForServerUsage: true, - CommonName: fmt.Sprintf("system:node:%s", nodeName), - Organizations: []string{user.NodesGroup}, - IPs: certIPs, - }) - if err != nil { - return nil, err - } - serverCertMgr.Start() - +func GenUseCertMgrAndTLSConfig(certificateMgr certificate.YurtCertificateManager) (*tls.Config, error) { // generate the TLS configuration based on the latest certificate rootCert, err := certmanager.GenCertPoolUseCA(certificateMgr.GetCaFile()) if err != nil { klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err) return nil, err } - tlsCfg, err := certmanager.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, rootCert, "server") + tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(certificateMgr.GetHubServerCert, rootCert, "server") if err != nil { return nil, err } - // waiting for the certificate is generated - _ = wait.PollUntil(5*time.Second, func() (bool, error) { - // keep polling until the certificate is signed - if serverCertMgr.Current() != nil { - return true, nil - } - klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetHubName()) - return false, nil - }, stopCh) - return tlsCfg, nil } diff --git a/pkg/yurthub/transport/transport.go b/pkg/yurthub/transport/transport.go index ed3e23a68ce..f7ce7505e0d 100644 --- a/pkg/yurthub/transport/transport.go +++ b/pkg/yurthub/transport/transport.go @@ -18,49 +18,48 @@ package transport import ( "crypto/tls" - "crypto/x509" "fmt" "net/http" - "os" "time" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces" + "github.com/openyurtio/openyurt/pkg/util/certmanager" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) // Interface is an transport interface for managing clients that used to connecting kube-apiserver type Interface interface { - // concurrent use by multiple goroutines // CurrentTransport get transport that used by load balancer + // and can be used by multiple goroutines concurrently. CurrentTransport() http.RoundTripper // BearerTransport returns transport for proxying request with bearer token in header BearerTransport() http.RoundTripper - // close all net connections that specified by address + // Close all net connections that specified by address Close(address string) } type transportManager struct { currentTransport *http.Transport bearerTransport *http.Transport - certManager interfaces.YurtCertificateManager + certManager certificate.YurtCertificateManager closeAll func() close func(string) stopCh <-chan struct{} } // NewTransportManager create a transport interface object. -func NewTransportManager(certMgr interfaces.YurtCertificateManager, stopCh <-chan struct{}) (Interface, error) { +func NewTransportManager(certMgr certificate.YurtCertificateManager, stopCh <-chan struct{}) (Interface, error) { caFile := certMgr.GetCaFile() if len(caFile) == 0 { return nil, fmt.Errorf("ca cert file was not prepared when new transport") } klog.V(2).Infof("use %s ca cert file to access remote server", caFile) - cfg, err := tlsConfig(certMgr, caFile) + cfg, err := tlsConfig(certMgr.GetAPIServerClientCert, caFile) if err != nil { klog.Errorf("could not get tls config when new transport, %v", err) return nil, err @@ -115,10 +114,10 @@ func (tm *transportManager) Close(address string) { } func (tm *transportManager) start() { - lastCert := tm.certManager.Current() + lastCert := tm.certManager.GetAPIServerClientCert() go wait.Until(func() { - curr := tm.certManager.Current() + curr := tm.certManager.GetAPIServerClientCert() if lastCert == nil && curr == nil { // maybe at yurthub startup, just wait for cert generated, do nothing @@ -143,48 +142,17 @@ func (tm *transportManager) start() { }, 10*time.Second, tm.stopCh) } -func tlsConfig(certMgr interfaces.YurtCertificateManager, caFile string) (*tls.Config, error) { - root, err := rootCertPool(caFile) +func tlsConfig(current func() *tls.Certificate, caFile string) (*tls.Config, error) { + // generate the TLS configuration based on the latest certificate + rootCert, err := certmanager.GenCertPoolUseCA(caFile) if err != nil { + klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err) return nil, err } - - tlsConfig := &tls.Config{ - // Can't use SSLv3 because of POODLE and BEAST - // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher - // Can't use TLSv1.1 because of RC4 cipher usage - MinVersion: tls.VersionTLS12, - RootCAs: root, - } - - if certMgr != nil { - tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - cert := certMgr.Current() - if cert == nil { - return &tls.Certificate{Certificate: nil}, nil - } - return cert, nil - } - } - - return tlsConfig, nil -} - -func rootCertPool(caFile string) (*x509.CertPool, error) { - if len(caFile) > 0 { - if caFileExists, err := util.FileExists(caFile); err != nil { - return nil, err - } else if caFileExists { - caData, err := os.ReadFile(caFile) - if err != nil { - return nil, err - } - - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(caData) - return certPool, nil - } + tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(current, rootCert, "client") + if err != nil { + return nil, err } - return nil, fmt.Errorf("failed to load ca file(%s)", caFile) + return tlsCfg, nil } diff --git a/pkg/yurthub/util/util.go b/pkg/yurthub/util/util.go index 39bdfc49fdf..b7617b07233 100644 --- a/pkg/yurthub/util/util.go +++ b/pkg/yurthub/util/util.go @@ -22,20 +22,14 @@ import ( "fmt" "io" "net/http" - "net/url" "os" "strings" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" apirequest "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -54,13 +48,7 @@ const ( WorkingModeCloud WorkingMode = "cloud" // WorkingModeEdge represents yurthub is working in edge mode, which means yurthub is deployed on the edge side. WorkingModeEdge WorkingMode = "edge" -) -const ( - // DefaultKubeletPairFilePath represents the default kubelet pair file path - DefaultKubeletPairFilePath = "/var/lib/kubelet/pki/kubelet-client-current.pem" - // DefaultKubeletRootCAFilePath represents the default kubelet ca file path - DefaultKubeletRootCAFilePath = "/etc/kubernetes/pki/ca.crt" // ProxyReqContentType represents request content type context key ProxyReqContentType ProxyKeyType = iota // ProxyRespContentType represents response content type context key @@ -305,93 +293,6 @@ func FileExists(filename string) (bool, error) { return true, nil } -// LoadKubeletRestClientConfig load *rest.Config for accessing healthyServer -func LoadKubeletRestClientConfig(healthyServer *url.URL, kubeletRootCAFilePath, kubeletPairFilePath string) (*rest.Config, error) { - if healthyServer == nil { - return nil, errors.New("healthyServer cannot be nil") - } - tlsClientConfig := rest.TLSClientConfig{} - if _, err := certutil.NewPool(kubeletRootCAFilePath); err != nil { - klog.Errorf("Expected to load root CA config from %s, but got err: %v", kubeletRootCAFilePath, err) - } else { - tlsClientConfig.CAFile = kubeletRootCAFilePath - } - - if can, _ := certutil.CanReadCertAndKey(kubeletPairFilePath, kubeletPairFilePath); !can { - return nil, fmt.Errorf("error reading %s, certificate and key must be supplied as a pair", kubeletPairFilePath) - } - tlsClientConfig.KeyFile = kubeletPairFilePath - tlsClientConfig.CertFile = kubeletPairFilePath - - return &rest.Config{ - Host: healthyServer.String(), - TLSClientConfig: tlsClientConfig, - }, nil -} - -func LoadRESTClientConfig(kubeconfig string) (*rest.Config, error) { - // Load structured kubeconfig data from the given path. - loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} - loadedConfig, err := loader.Load() - if err != nil { - return nil, err - } - // Flatten the loaded data to a particular restclient.Config based on the current context. - return clientcmd.NewNonInteractiveClientConfig( - *loadedConfig, - loadedConfig.CurrentContext, - &clientcmd.ConfigOverrides{}, - loader, - ).ClientConfig() -} - -func LoadKubeConfig(kubeconfig string) (*clientcmdapi.Config, error) { - loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} - loadedConfig, err := loader.Load() - if err != nil { - return nil, err - } - - return loadedConfig, nil -} - -func CreateKubeConfigFile(kubeClientConfig *rest.Config, kubeconfigPath string) error { - if kubeClientConfig == nil { - return fmt.Errorf("kube client config cannot be nil") - } - // Get the CA data from the bootstrap client config. - caFile, caData := kubeClientConfig.CAFile, []byte{} - if len(caFile) == 0 { - caData = kubeClientConfig.CAData - } - - // Build resulting kubeconfig. - kubeconfigData := clientcmdapi.Config{ - // Define a cluster stanza based on the bootstrap kubeconfig. - Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { - Server: kubeClientConfig.Host, - InsecureSkipTLSVerify: kubeClientConfig.Insecure, - CertificateAuthority: caFile, - CertificateAuthorityData: caData, - }}, - // Define auth based on the obtained client cert. - AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": { - ClientCertificate: kubeClientConfig.CertFile, - ClientKey: kubeClientConfig.KeyFile, - }}, - // Define a context that connects the auth info and cluster, and set it as the default - Contexts: map[string]*clientcmdapi.Context{"default-context": { - Cluster: "default-cluster", - AuthInfo: "default-auth", - Namespace: "default", - }}, - CurrentContext: "default-context", - } - - // Marshal to disk - return clientcmd.WriteToFile(kubeconfigData, kubeconfigPath) -} - // gzipReaderCloser will gunzip the data if response header // contains Content-Encoding=gzip header. type gzipReaderCloser struct { @@ -432,7 +333,6 @@ func NewGZipReaderCloser(header http.Header, body io.ReadCloser, req *http.Reque } func ParseTenantNs(certOrg string) string { - if !strings.Contains(certOrg, "openyurt:tenant:") { return "" } @@ -441,20 +341,15 @@ func ParseTenantNs(certOrg string) string { } func ParseTenantNsFromOrgs(orgs []string) string { - + var ns string if len(orgs) == 0 { - - return "" + return ns } - ns := "" for _, v := range orgs { - - tns := ParseTenantNs(v) - - if tns != "" { - ns = tns - break + ns := ParseTenantNs(v) + if len(ns) != 0 { + return ns } } @@ -462,7 +357,6 @@ func ParseTenantNsFromOrgs(orgs []string) string { } func ParseBearerToken(token string) string { - if token == "" { return "" } diff --git a/pkg/yurthub/util/util_test.go b/pkg/yurthub/util/util_test.go index ef808a23c09..a6f7f9103c1 100644 --- a/pkg/yurthub/util/util_test.go +++ b/pkg/yurthub/util/util_test.go @@ -29,25 +29,6 @@ import ( "testing" apirequest "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/client-go/rest" -) - -var ( - testCaCert = []byte(`-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJANXr+UzRFq4TMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwIBcNMTcwNDI2MjMyNzMyWhgPMjExNzA0 -MDIyMzI3MzJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTEwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAqvbkN4RShH1rL37JFp4fZPnn0JUhVWWsrP8NOomJ -pXdBDUMGWuEQIsZ1Gf9JrCQLu6ooRyHSKRFpAVbMQ3ABJwIDAQABo1AwTjAdBgNV -HQ4EFgQUEGBc6YYheEZ/5MhwqSUYYPYRj2MwHwYDVR0jBBgwFoAUEGBc6YYheEZ/ -5MhwqSUYYPYRj2MwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAIyNmznk -5dgJY52FppEEcfQRdS5k4XFPc22SHPcz77AHf5oWZ1WG9VezOZZPp8NCiFDDlDL8 -yma33a5eMyTjLD8= ------END CERTIFICATE-----`) ) func TestContext(t *testing.T) { @@ -425,192 +406,6 @@ func TestNewGZipReaderCloser(t *testing.T) { } } -func TestCreateKubeConfigFile(t *testing.T) { - dir, err := ioutil.TempDir("", "yurthub-util-kubeconfig") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - testKubeConfigPath := dir + "kube.config" - type args struct { - kubeClientConfig *rest.Config - kubeconfigPath string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no input return error", args{}, true}, - {"no legal input return error", args{kubeClientConfig: new(rest.Config)}, true}, - {"legal input with no error", args{kubeClientConfig: &rest.Config{ - TLSClientConfig: rest.TLSClientConfig{ - CAData: testCaCert, - CertFile: "/tmp/not-exist-path", - KeyFile: "/tmp/not-exist-path", - }, - }, kubeconfigPath: testKubeConfigPath}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := CreateKubeConfigFile(tt.args.kubeClientConfig, tt.args.kubeconfigPath); (err != nil) != tt.wantErr { - t.Errorf("CreateKubeConfigFile() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestLoadKubeConfig(t *testing.T) { - type args struct { - kubeconfig string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no config path, return default config", args{}, false}, - {"illegal config path, return error", args{"/tmp/not-exist-config-path"}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := LoadKubeConfig(tt.args.kubeconfig) - if (err != nil) != tt.wantErr { - t.Errorf("LoadKubeConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func TestLoadRESTClientConfig(t *testing.T) { - dir, err := ioutil.TempDir("", "yurthub-util-kubeconfig") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - testKubeConfigPath := dir + "kube.config" - if err = ioutil.WriteFile(testKubeConfigPath, []byte(`apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmVENDQXllZ0F3SUJBZ0lVRkJsNGdVb3FaRFAvd1VKRG4zNy9WSjl1cEQwd0RRWUpLb1pJaHZjTkFRRUYKQlFBd2ZqRUxNQWtHQTFVRUJoTUNSMEl4RHpBTkJnTlZCQWdNQmt4dmJtUnZiakVQTUEwR0ExVUVCd3dHVEc5dQpaRzl1TVJnd0ZnWURWUVFLREE5SGJHOWlZV3dnVTJWamRYSnBkSGt4RmpBVUJnTlZCQXNNRFVsVUlFUmxjR0Z5CmRHMWxiblF4R3pBWkJnTlZCQU1NRW5SbGMzUXRZMlZ5ZEdsbWFXTmhkR1V0TURBZUZ3MHlNREF6TURJeE9UTTMKTURCYUZ3MHlNVEF6TURJeE9UTTNNREJhTUlHSU1Rc3dDUVlEVlFRR0V3SlZVekVUTUJFR0ExVUVDQk1LUTJGcwphV1p2Y201cFlURVdNQlFHQTFVRUJ4TU5VMkZ1SUVaeVlXNWphWE5qYnpFZE1Cc0dBMVVFQ2hNVVJYaGhiWEJzClpTQkRiMjF3WVc1NUxDQk1URU14RXpBUkJnTlZCQXNUQ2s5d1pYSmhkR2x2Ym5NeEdEQVdCZ05WQkFNVEQzZDMKZHk1bGVHRnRjR3hsTG1OdmJUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU1pUgpETnBtd1RJQ0ZyK1AxNmZLRFZqYk5DelNqV3ErTVR1OHZBZlM2R3JMcEJUVUVlKzZ6VnF4VXphL2ZaZW54bzhPCnVjVjJKVFV2NUo0bmtUL3ZHNlFtL21Ub1ZKNHZRekxRNWpSMnc3di83Y2Yzb1dDd1RBS1VhZmdvNi9HYTk1Z24KbFFCMytGZDhzeTk2emZGci83d0RTTVBQdWVSNWtTRmF4K2NFZDMwd3d2NU83dFdqMHJvMW1yeExzc0Jsd1BhUgpabHpra3Z4QllUeldDcUtac1drdFFsWGNpcWxGU29zMHVhN3V2d3FLTjVDVHhmQy94b3lNeHg5a2ZabTdCelBOClpEcVlNRncySGlXZEVpTHpJNGpqK0doMEQ1dDQ3dG52bHBVTWloY1g5eDBqUDYvK2huZmNROEdBUDJqUi9CWFkKNVlaUlJZNzBMaUNYUGV2bFJBRUNBd0VBQWFPQnFUQ0JwakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SFFZRFZSME9CQllFCkZPb2lFK2toN2dHRHB5eDBLWnVDYzFscmxUUktNQjhHQTFVZEl3UVlNQmFBRk50b3N2R2xwRFVzYjlKd2NSY1gKcTM3TDUyVlRNQ2NHQTFVZEVRUWdNQjZDQzJWNFlXMXdiR1V1WTI5dGdnOTNkM2N1WlhoaGJYQnNaUzVqYjIwdwpEUVlKS29aSWh2Y05BUUVGQlFBRFFRQXc2bXhRT05BRDJzaXZmeklmMWVERmQ2TFU3YUUrTW5rZGxFUWpqUENpCnRsVUlURkl1TzNYYXZJU3VwUDZWOXdFMGIxd1RGMXBUbFZXQXJmLzBZUVhzCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://127.0.0.1 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kubernetes-admin - name: kubernetes-admin@kubernetes -current-context: kubernetes-admin@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-admin - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNSekNDQWZHZ0F3SUJBZ0lKQUxNYjdlY01JazNNTUEwR0NTcUdTSWIzRFFFQkN3VUFNSDR4Q3pBSkJnTlYKQkFZVEFrZENNUTh3RFFZRFZRUUlEQVpNYjI1a2IyNHhEekFOQmdOVkJBY01Ca3h2Ym1SdmJqRVlNQllHQTFVRQpDZ3dQUjJ4dlltRnNJRk5sWTNWeWFYUjVNUll3RkFZRFZRUUxEQTFKVkNCRVpYQmhjblJ0Wlc1ME1Sc3dHUVlEClZRUUREQkowWlhOMExXTmxjblJwWm1sallYUmxMVEF3SUJjTk1UY3dOREkyTWpNeU5qVXlXaGdQTWpFeE56QTAKTURJeU16STJOVEphTUg0eEN6QUpCZ05WQkFZVEFrZENNUTh3RFFZRFZRUUlEQVpNYjI1a2IyNHhEekFOQmdOVgpCQWNNQmt4dmJtUnZiakVZTUJZR0ExVUVDZ3dQUjJ4dlltRnNJRk5sWTNWeWFYUjVNUll3RkFZRFZRUUxEQTFKClZDQkVaWEJoY25SdFpXNTBNUnN3R1FZRFZRUUREQkowWlhOMExXTmxjblJwWm1sallYUmxMVEF3WERBTkJna3EKaGtpRzl3MEJBUUVGQUFOTEFEQklBa0VBdEJNYTdOV3B2M0JWbEtUQ1BHTy9MRXNndUtxV0hCdEt6d2VNWTJDVgp0QUwxclFtOTEzaHVoeEY5dythaTc2S1EzTUhLNUlWbkxKallZQTVNelAySDVRSURBUUFCbzFBd1RqQWRCZ05WCkhRNEVGZ1FVMjJpeThhV2tOU3h2MG5CeEZ4ZXJmc3ZuWlZNd0h3WURWUjBqQkJnd0ZvQVUyMml5OGFXa05TeHYKMG5CeEZ4ZXJmc3ZuWlZNd0RBWURWUjBUQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCQVFzRkFBTkJBRU9lZkdiVgpOY0h4a2xhVzA2dzZPQllKUHdwSWhDVm96QzFxZHhHWDFkZzhWa0VLempPempncVZEMzBtNTlPRm1TbEJtSHNsCm5rVkE2d3lPU0RZQmYzbz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlCVXdJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0FUMHdnZ0U1QWdFQUFrRUF0Qk1hN05XcHYzQlZsS1RDClBHTy9MRXNndUtxV0hCdEt6d2VNWTJDVnRBTDFyUW05MTNodWh4Rjl3K2FpNzZLUTNNSEs1SVZuTEpqWVlBNU0KelAySDVRSURBUUFCQWtBUzlCZlhhYjNPS3BLM2JJZ05OeXArRFFKS3JablRKNFErT2pzcWtwWHZObHRQSm9zZgpHOEdzaUt1L3ZBdDRIR3FJM2VVNzdOdlJJK21MNE1uSFJtWEJBaUVBM3FNNEZBdEtTUkJiY0p6UHh4TEVVU3dnClhTQ2Nvc0NrdGJrWHZwWXJTMzBDSVFEUER4Z3Fsd0RFSlEwdUt1SGtaSTM4L1NQV1dxZlVta2Vjd2xicFhBQksKaVFJZ1pYMDhEQThWZnZjQTUvWGoxWmpkZXk5RlZZNlBPTFhlbjZSUGlhYkU5N1VDSUNwNmVVVzdodCsyamphcgplMzVFbHRDUkNqb2VqUkhUdU45VEMwdUNvVmlwQWlBWGFKSXgvUTQ3dkd3aXc2WThLWHNOVTZ5NTRnVGJPU3hYCjU0THpITmsvK1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=`), 0600); err != nil { - t.Fatalf("failed to write test kube config to tmp dir. err:%v", err) - } - type args struct { - kubeconfig string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no config path, return error", args{}, true}, - {"illegal config path, return error", args{"/tmp/not-exist-config-path"}, true}, - {"legal config path, no error", args{testKubeConfigPath}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := LoadRESTClientConfig(tt.args.kubeconfig) - if (err != nil) != tt.wantErr { - t.Errorf("LoadRESTClientConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - }) - } -} - -func TestLoadKubeletRestClientConfig(t *testing.T) { - dir, err := ioutil.TempDir("", "yurthub-util-load-kubelet-rest-client-config") - if err != nil { - t.Fatalf("Unable to create the test directory %q: %v", dir, err) - } - defer func() { - if err := os.RemoveAll(dir); err != nil { - t.Errorf("Unable to clean up test directory %q: %v", dir, err) - } - }() - legalKubeConfigFile := dir + "legal.config" - if err = ioutil.WriteFile(legalKubeConfigFile, []byte(`-----BEGIN CERTIFICATE----- -MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV -BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE -CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MRswGQYD -VQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwIBcNMTcwNDI2MjMyNjUyWhgPMjExNzA0 -MDIyMzI2NTJaMH4xCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNV -BAcMBkxvbmRvbjEYMBYGA1UECgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1J -VCBEZXBhcnRtZW50MRswGQYDVQQDDBJ0ZXN0LWNlcnRpZmljYXRlLTAwXDANBgkq -hkiG9w0BAQEFAANLADBIAkEAtBMa7NWpv3BVlKTCPGO/LEsguKqWHBtKzweMY2CV -tAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5MzP2H5QIDAQABo1AwTjAdBgNV -HQ4EFgQU22iy8aWkNSxv0nBxFxerfsvnZVMwHwYDVR0jBBgwFoAU22iy8aWkNSxv -0nBxFxerfsvnZVMwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEOefGbV -NcHxklaW06w6OBYJPwpIhCVozC1qdxGX1dg8VkEKzjOzjgqVD30m59OFmSlBmHsl -nkVA6wyOSDYBf3o= ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAtBMa7NWpv3BVlKTC -PGO/LEsguKqWHBtKzweMY2CVtAL1rQm913huhxF9w+ai76KQ3MHK5IVnLJjYYA5M -zP2H5QIDAQABAkAS9BfXab3OKpK3bIgNNyp+DQJKrZnTJ4Q+OjsqkpXvNltPJosf -G8GsiKu/vAt4HGqI3eU77NvRI+mL4MnHRmXBAiEA3qM4FAtKSRBbcJzPxxLEUSwg -XSCcosCktbkXvpYrS30CIQDPDxgqlwDEJQ0uKuHkZI38/SPWWqfUmkecwlbpXABK -iQIgZX08DA8VfvcA5/Xj1Zjdey9FVY6POLXen6RPiabE97UCICp6eUW7ht+2jjar -e35EltCRCjoejRHTuN9TC0uCoVipAiAXaJIx/Q47vGwiw6Y8KXsNU6y54gTbOSxX -54LzHNk/+Q== ------END RSA PRIVATE KEY-----`), 0600); err != nil { - t.Fatalf("write legal kube config to file. err:%v", err) - } - illegalKubeConfigFile := dir + "illegal.config" - if err = ioutil.WriteFile(illegalKubeConfigFile, []byte(`something illegal`), 0600); err != nil { - t.Fatalf("write illegal kube config to file. err:%v", err) - } - type args struct { - healthyServer *url.URL - kubeletRootCAFilePath string - kubeletPairFilePath string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"no input, want error", args{}, true}, - {"no root ca, want error", args{healthyServer: new(url.URL)}, true}, - {"no legal root ca, want error", args{healthyServer: new(url.URL), kubeletRootCAFilePath: illegalKubeConfigFile}, true}, - {"legal root ca, no error", args{healthyServer: new(url.URL), kubeletPairFilePath: legalKubeConfigFile}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := LoadKubeletRestClientConfig(tt.args.healthyServer, tt.args.kubeletRootCAFilePath, tt.args.kubeletPairFilePath) - if (err != nil) != tt.wantErr { - t.Errorf("LoadKubeletRestClientConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - - }) - } -} - func TestReqInfoString(t *testing.T) { type args struct { info *apirequest.RequestInfo