-
Notifications
You must be signed in to change notification settings - Fork 407
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
change yurthub's protocol from http to https #368
Changes from 8 commits
2a9460f
0a0e313
4775c54
a3dd98b
2b53769
f2a30e4
229bb3f
55b3ea2
c516d4d
d04af20
8b8f874
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,7 @@ limitations under the License. | |
package config | ||
|
||
import ( | ||
"crypto/x509" | ||
"fmt" | ||
"net" | ||
"net/url" | ||
|
@@ -28,31 +29,43 @@ import ( | |
"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" | ||
"github.com/openyurtio/openyurt/pkg/yurthub/storage/factory" | ||
|
||
"k8s.io/client-go/informers" | ||
"k8s.io/client-go/kubernetes" | ||
"k8s.io/klog" | ||
) | ||
|
||
// YurtHubConfiguration represents configuration of yurthub | ||
type YurtHubConfiguration struct { | ||
LBMode string | ||
RemoteServers []*url.URL | ||
YurtHubServerAddr string | ||
YurtHubProxyServerAddr string | ||
YurtHubProxyServerDummyAddr string | ||
GCFrequency int | ||
CertMgrMode string | ||
NodeName string | ||
HeartbeatFailedRetry int | ||
HeartbeatHealthyThreshold int | ||
HeartbeatTimeoutSeconds int | ||
MaxRequestInFlight int | ||
JoinToken string | ||
RootDir string | ||
EnableProfiling bool | ||
EnableDummyIf bool | ||
EnableIptables bool | ||
HubAgentDummyIfName string | ||
StorageWrapper cachemanager.StorageWrapper | ||
SerializerManager *serializer.SerializerManager | ||
LBMode string | ||
RemoteServers []*url.URL | ||
YurtHubServerAddr string | ||
YurtHubProxyServerAddr string | ||
YurtHubProxyServerSecureAddr string | ||
YurtHubProxyServerDummyAddr string | ||
YurtHubProxyServerSecureDummyAddr string | ||
GCFrequency int | ||
CertMgrMode string | ||
NodeName string | ||
HeartbeatFailedRetry int | ||
HeartbeatHealthyThreshold int | ||
HeartbeatTimeoutSeconds int | ||
MaxRequestInFlight int | ||
JoinToken string | ||
RootDir string | ||
EnableProfiling bool | ||
EnableDummyIf bool | ||
EnableIptables bool | ||
HubAgentDummyIfName string | ||
StorageWrapper cachemanager.StorageWrapper | ||
SerializerManager *serializer.SerializerManager | ||
CertFile string | ||
KeyFile string | ||
KubeConfig string | ||
RootCert *x509.CertPool | ||
Client *kubernetes.Clientset | ||
CertDNSNames []string | ||
CertIPs []net.IP | ||
SharedInformerFactory informers.SharedInformerFactory | ||
} | ||
|
||
// Complete converts *options.YurtHubOptions to *YurtHubConfiguration | ||
|
@@ -72,28 +85,32 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { | |
|
||
hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort) | ||
proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort) | ||
proxySecureServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxySecurePort) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yurthub's https also need to listen on There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The latest code has been commited. Yurthub's https also listens on HubAgentDummyIfIP address |
||
proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort) | ||
proxySecureServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxySecurePort) | ||
cfg := &YurtHubConfiguration{ | ||
LBMode: options.LBMode, | ||
RemoteServers: us, | ||
YurtHubServerAddr: hubServerAddr, | ||
YurtHubProxyServerAddr: proxyServerAddr, | ||
YurtHubProxyServerDummyAddr: proxyServerDummyAddr, | ||
GCFrequency: options.GCFrequency, | ||
CertMgrMode: options.CertMgrMode, | ||
NodeName: options.NodeName, | ||
HeartbeatFailedRetry: options.HeartbeatFailedRetry, | ||
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, | ||
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, | ||
MaxRequestInFlight: options.MaxRequestInFlight, | ||
JoinToken: options.JoinToken, | ||
RootDir: options.RootDir, | ||
EnableProfiling: options.EnableProfiling, | ||
EnableDummyIf: options.EnableDummyIf, | ||
EnableIptables: options.EnableIptables, | ||
HubAgentDummyIfName: options.HubAgentDummyIfName, | ||
StorageWrapper: storageWrapper, | ||
SerializerManager: serializerManager, | ||
LBMode: options.LBMode, | ||
RemoteServers: us, | ||
YurtHubServerAddr: hubServerAddr, | ||
YurtHubProxyServerAddr: proxyServerAddr, | ||
YurtHubProxyServerSecureAddr: proxySecureServerAddr, | ||
YurtHubProxyServerDummyAddr: proxyServerDummyAddr, | ||
YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr, | ||
GCFrequency: options.GCFrequency, | ||
CertMgrMode: options.CertMgrMode, | ||
NodeName: options.NodeName, | ||
HeartbeatFailedRetry: options.HeartbeatFailedRetry, | ||
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, | ||
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, | ||
MaxRequestInFlight: options.MaxRequestInFlight, | ||
JoinToken: options.JoinToken, | ||
RootDir: options.RootDir, | ||
EnableProfiling: options.EnableProfiling, | ||
EnableDummyIf: options.EnableDummyIf, | ||
EnableIptables: options.EnableIptables, | ||
HubAgentDummyIfName: options.HubAgentDummyIfName, | ||
StorageWrapper: storageWrapper, | ||
SerializerManager: serializerManager, | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the establishment and configuration of TLS may be better done by yurthub, users do not need to pay attention to the underlying certificate configuration by default. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's a good idea. I will improve it later. |
||
|
||
return cfg, nil | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,6 +38,7 @@ type YurtHubOptions struct { | |
YurtHubHost string | ||
YurtHubPort string | ||
YurtHubProxyPort string | ||
YurtHubProxySecurePort string | ||
GCFrequency int | ||
CertMgrMode string | ||
NodeName string | ||
|
@@ -61,6 +62,7 @@ type YurtHubOptions struct { | |
func NewYurtHubOptions() *YurtHubOptions { | ||
o := &YurtHubOptions{ | ||
YurtHubHost: "127.0.0.1", | ||
YurtHubProxySecurePort: "10260", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe 10260 is used by k8s, how about 10268? |
||
YurtHubProxyPort: "10261", | ||
YurtHubPort: "10267", | ||
GCFrequency: 120, | ||
|
@@ -112,6 +114,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { | |
fs.StringVar(&o.YurtHubHost, "bind-address", o.YurtHubHost, "the IP address on which to listen for the --serve-port port.") | ||
fs.StringVar(&o.YurtHubPort, "serve-port", o.YurtHubPort, "the port on which to serve HTTP requests(like profiling, metrics) for hub agent.") | ||
fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver") | ||
fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver") | ||
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"") | ||
fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.") | ||
fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).") | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -147,8 +147,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { | |
klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. please add |
||
} | ||
|
||
klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr) | ||
s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler) | ||
klog.Infof("%d. new %s server and begin to serve, proxy server: %s, secure proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubProxyServerSecureAddr, cfg.YurtHubServerAddr) | ||
s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler, stopCh) | ||
if err != nil { | ||
klog.Errorf("could not create hub server, %v", err) | ||
return err | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
/* | ||
Copyright 2020 The OpenYurt Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package certmanager | ||
|
||
import ( | ||
"crypto/tls" | ||
"crypto/x509" | ||
"crypto/x509/pkix" | ||
"fmt" | ||
"github.com/openyurtio/openyurt/pkg/projectinfo" | ||
certificates "k8s.io/api/certificates/v1beta1" | ||
"k8s.io/client-go/kubernetes" | ||
clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" | ||
"k8s.io/client-go/util/certificate" | ||
"k8s.io/klog/v2" | ||
) | ||
|
||
const ( | ||
YurtHubServerCSROrg = "system:nodes" | ||
YurtHubServerCSRCN = "kube-apiserver-kubelet-client" | ||
) | ||
|
||
// NewYurtHubServerCertManager creates a certificate manager for | ||
// the yurthub-server | ||
func NewYurtHubServerCertManager( | ||
clientset kubernetes.Interface, | ||
certDir string) (certificate.Manager, error) { | ||
|
||
klog.Infof("subject of yurthub server certificate") | ||
return newCertManager( | ||
clientset, | ||
projectinfo.GetHubName(), | ||
certDir, | ||
YurtHubServerCSRCN, | ||
[]string{YurtHubServerCSROrg}) | ||
} | ||
|
||
// NewCertManager creates a certificate manager that will generates a | ||
// certificate by sending a csr to the apiserver | ||
func newCertManager( | ||
clientset kubernetes.Interface, | ||
componentName, | ||
certDir, | ||
commonName string, | ||
organizations []string) (certificate.Manager, error) { | ||
certificateStore, err := | ||
certificate.NewFileStore(componentName, certDir, certDir, "", "") | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to initialize the server certificate store: %v", err) | ||
} | ||
|
||
getTemplate := func() *x509.CertificateRequest { | ||
return &x509.CertificateRequest{ | ||
Subject: pkix.Name{ | ||
CommonName: commonName, | ||
Organization: organizations, | ||
}, | ||
} | ||
} | ||
|
||
certManager, err := certificate.NewManager(&certificate.Config{ | ||
ClientFn: func(current *tls.Certificate) (clicert.CertificateSigningRequestInterface, error) { | ||
return clientset.CertificatesV1beta1().CertificateSigningRequests(), nil | ||
}, | ||
SignerName: certificates.KubeAPIServerClientKubeletSignerName, | ||
GetTemplate: getTemplate, | ||
Usages: []certificates.KeyUsage{ | ||
certificates.UsageAny, | ||
}, | ||
CertificateStore: certificateStore, | ||
}) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err) | ||
} | ||
|
||
return certManager, nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
/* | ||
Copyright 2020 The OpenYurt Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package pki | ||
|
||
import ( | ||
"crypto/tls" | ||
"crypto/x509" | ||
"errors" | ||
"fmt" | ||
"io/ioutil" | ||
"os" | ||
|
||
"k8s.io/client-go/util/certificate" | ||
) | ||
|
||
// GenTGenTLSConfigUseCertMgrAndCertPool generates a TLS configuration | ||
// using the given certificate manager and x509 CertPool | ||
func GenTLSConfigUseCertMgrAndCertPool( | ||
m certificate.Manager, | ||
root *x509.CertPool) (*tls.Config, error) { | ||
tlsConfig := &tls.Config{ | ||
// Can't use SSLv3 because of POODLE and BEAST | ||
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher | ||
// Can't use TLSv1.1 because of RC4 cipher usage | ||
MinVersion: tls.VersionTLS12, | ||
RootCAs: root, | ||
ClientAuth: tls.RequireAnyClientCert, | ||
} | ||
|
||
tlsConfig.GetClientCertificate = | ||
func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { | ||
cert := m.Current() | ||
if cert == nil { | ||
return &tls.Certificate{Certificate: nil}, nil | ||
} | ||
return cert, nil | ||
} | ||
tlsConfig.GetCertificate = | ||
func(*tls.ClientHelloInfo) (*tls.Certificate, error) { | ||
cert := m.Current() | ||
if cert == nil { | ||
return &tls.Certificate{Certificate: nil}, nil | ||
} | ||
return cert, nil | ||
} | ||
|
||
return tlsConfig, nil | ||
} | ||
|
||
// GenCertPoolUseCA generates a x509 CertPool based on the given CA file | ||
func GenCertPoolUseCA(caFile string) (*x509.CertPool, error) { | ||
if caFile == "" { | ||
return nil, errors.New("CA file is not set") | ||
} | ||
|
||
if _, err := os.Stat(caFile); err != nil { | ||
if os.IsNotExist(err) { | ||
return nil, fmt.Errorf("CA file(%s) doesn't exist", caFile) | ||
} | ||
return nil, fmt.Errorf("fail to stat the CA file(%s): %s", caFile, err) | ||
} | ||
|
||
caData, err := ioutil.ReadFile(caFile) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
certPool := x509.NewCertPool() | ||
certPool.AppendCertsFromPEM(caData) | ||
return certPool, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The following fields should be deleted?