Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

change yurthub's protocol from http to https #368

Closed
Closed
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 57 additions & 40 deletions cmd/yurthub/app/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package config

import (
"crypto/x509"
"fmt"
"net"
"net/url"
Expand All @@ -28,31 +29,43 @@ import (
"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer"
"github.com/openyurtio/openyurt/pkg/yurthub/storage/factory"

"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"
)

// YurtHubConfiguration represents configuration of yurthub
type YurtHubConfiguration struct {
LBMode string
RemoteServers []*url.URL
YurtHubServerAddr string
YurtHubProxyServerAddr string
YurtHubProxyServerDummyAddr string
GCFrequency int
CertMgrMode string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
HeartbeatTimeoutSeconds int
MaxRequestInFlight int
JoinToken string
RootDir string
EnableProfiling bool
EnableDummyIf bool
EnableIptables bool
HubAgentDummyIfName string
StorageWrapper cachemanager.StorageWrapper
SerializerManager *serializer.SerializerManager
LBMode string
RemoteServers []*url.URL
YurtHubServerAddr string
YurtHubProxyServerAddr string
YurtHubProxyServerSecureAddr string
YurtHubProxyServerDummyAddr string
YurtHubProxyServerSecureDummyAddr string
GCFrequency int
CertMgrMode string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
HeartbeatTimeoutSeconds int
MaxRequestInFlight int
JoinToken string
RootDir string
EnableProfiling bool
EnableDummyIf bool
EnableIptables bool
HubAgentDummyIfName string
StorageWrapper cachemanager.StorageWrapper
SerializerManager *serializer.SerializerManager
CertFile string
KeyFile string
KubeConfig string
RootCert *x509.CertPool
Client *kubernetes.Clientset
CertDNSNames []string
CertIPs []net.IP
SharedInformerFactory informers.SharedInformerFactory
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The following fields should be deleted?

	CertFile                          string
	KeyFile                           string
	KubeConfig                        string
	RootCert                          *x509.CertPool
	Client                            *kubernetes.Clientset
	CertDNSNames                      []string
	CertIPs                           []net.IP
	SharedInformerFactory             informers.SharedInformerFactory

}

// Complete converts *options.YurtHubOptions to *YurtHubConfiguration
Expand All @@ -72,28 +85,32 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {

hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort)
proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort)
proxySecureServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxySecurePort)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yurthub's https also need to listen on HubAgentDummyIfIP address

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The latest code has been commited. Yurthub's https also listens on HubAgentDummyIfIP address

proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort)
proxySecureServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxySecurePort)
cfg := &YurtHubConfiguration{
LBMode: options.LBMode,
RemoteServers: us,
YurtHubServerAddr: hubServerAddr,
YurtHubProxyServerAddr: proxyServerAddr,
YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
GCFrequency: options.GCFrequency,
CertMgrMode: options.CertMgrMode,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds,
MaxRequestInFlight: options.MaxRequestInFlight,
JoinToken: options.JoinToken,
RootDir: options.RootDir,
EnableProfiling: options.EnableProfiling,
EnableDummyIf: options.EnableDummyIf,
EnableIptables: options.EnableIptables,
HubAgentDummyIfName: options.HubAgentDummyIfName,
StorageWrapper: storageWrapper,
SerializerManager: serializerManager,
LBMode: options.LBMode,
RemoteServers: us,
YurtHubServerAddr: hubServerAddr,
YurtHubProxyServerAddr: proxyServerAddr,
YurtHubProxyServerSecureAddr: proxySecureServerAddr,
YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr,
GCFrequency: options.GCFrequency,
CertMgrMode: options.CertMgrMode,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds,
MaxRequestInFlight: options.MaxRequestInFlight,
JoinToken: options.JoinToken,
RootDir: options.RootDir,
EnableProfiling: options.EnableProfiling,
EnableDummyIf: options.EnableDummyIf,
EnableIptables: options.EnableIptables,
HubAgentDummyIfName: options.HubAgentDummyIfName,
StorageWrapper: storageWrapper,
SerializerManager: serializerManager,
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the establishment and configuration of TLS may be better done by yurthub, users do not need to pay attention to the underlying certificate configuration by default.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a good idea. I will improve it later.


return cfg, nil
Expand Down
3 changes: 3 additions & 0 deletions cmd/yurthub/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type YurtHubOptions struct {
YurtHubHost string
YurtHubPort string
YurtHubProxyPort string
YurtHubProxySecurePort string
GCFrequency int
CertMgrMode string
NodeName string
Expand All @@ -61,6 +62,7 @@ type YurtHubOptions struct {
func NewYurtHubOptions() *YurtHubOptions {
o := &YurtHubOptions{
YurtHubHost: "127.0.0.1",
YurtHubProxySecurePort: "10260",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe 10260 is used by k8s, how about 10268?

YurtHubProxyPort: "10261",
YurtHubPort: "10267",
GCFrequency: 120,
Expand Down Expand Up @@ -112,6 +114,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.YurtHubHost, "bind-address", o.YurtHubHost, "the IP address on which to listen for the --serve-port port.")
fs.StringVar(&o.YurtHubPort, "serve-port", o.YurtHubPort, "the port on which to serve HTTP requests(like profiling, metrics) for hub agent.")
fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver")
fs.StringVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver")
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.")
fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
Expand Down
4 changes: 2 additions & 2 deletions cmd/yurthub/app/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please add YurtHubProxyServerSecureDummyAddr here

}

klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr)
s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler)
klog.Infof("%d. new %s server and begin to serve, proxy server: %s, secure proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubProxyServerSecureAddr, cfg.YurtHubServerAddr)
s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler, stopCh)
if err != nil {
klog.Errorf("could not create hub server, %v", err)
return err
Expand Down
91 changes: 91 additions & 0 deletions pkg/yurthub/pki/certmanager/certmanager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
Copyright 2020 The OpenYurt Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certmanager

import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"github.com/openyurtio/openyurt/pkg/projectinfo"
certificates "k8s.io/api/certificates/v1beta1"
"k8s.io/client-go/kubernetes"
clicert "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
"k8s.io/client-go/util/certificate"
"k8s.io/klog/v2"
)

const (
YurtHubServerCSROrg = "system:nodes"
YurtHubServerCSRCN = "kube-apiserver-kubelet-client"
)

// NewYurtHubServerCertManager creates a certificate manager for
// the yurthub-server
func NewYurtHubServerCertManager(
clientset kubernetes.Interface,
certDir string) (certificate.Manager, error) {

klog.Infof("subject of yurthub server certificate")
return newCertManager(
clientset,
projectinfo.GetHubName(),
certDir,
YurtHubServerCSRCN,
[]string{YurtHubServerCSROrg})
}

// NewCertManager creates a certificate manager that will generates a
// certificate by sending a csr to the apiserver
func newCertManager(
clientset kubernetes.Interface,
componentName,
certDir,
commonName string,
organizations []string) (certificate.Manager, error) {
certificateStore, err :=
certificate.NewFileStore(componentName, certDir, certDir, "", "")
if err != nil {
return nil, fmt.Errorf("failed to initialize the server certificate store: %v", err)
}

getTemplate := func() *x509.CertificateRequest {
return &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: commonName,
Organization: organizations,
},
}
}

certManager, err := certificate.NewManager(&certificate.Config{
ClientFn: func(current *tls.Certificate) (clicert.CertificateSigningRequestInterface, error) {
return clientset.CertificatesV1beta1().CertificateSigningRequests(), nil
},
SignerName: certificates.KubeAPIServerClientKubeletSignerName,
GetTemplate: getTemplate,
Usages: []certificates.KeyUsage{
certificates.UsageAny,
},
CertificateStore: certificateStore,
})
if err != nil {
return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err)
}

return certManager, nil
}
85 changes: 85 additions & 0 deletions pkg/yurthub/pki/pki.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
Copyright 2020 The OpenYurt Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pki

import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"os"

"k8s.io/client-go/util/certificate"
)

// GenTGenTLSConfigUseCertMgrAndCertPool generates a TLS configuration
// using the given certificate manager and x509 CertPool
func GenTLSConfigUseCertMgrAndCertPool(
m certificate.Manager,
root *x509.CertPool) (*tls.Config, error) {
tlsConfig := &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
RootCAs: root,
ClientAuth: tls.RequireAnyClientCert,
}

tlsConfig.GetClientCertificate =
func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert := m.Current()
if cert == nil {
return &tls.Certificate{Certificate: nil}, nil
}
return cert, nil
}
tlsConfig.GetCertificate =
func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
cert := m.Current()
if cert == nil {
return &tls.Certificate{Certificate: nil}, nil
}
return cert, nil
}

return tlsConfig, nil
}

// GenCertPoolUseCA generates a x509 CertPool based on the given CA file
func GenCertPoolUseCA(caFile string) (*x509.CertPool, error) {
if caFile == "" {
return nil, errors.New("CA file is not set")
}

if _, err := os.Stat(caFile); err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("CA file(%s) doesn't exist", caFile)
}
return nil, fmt.Errorf("fail to stat the CA file(%s): %s", caFile, err)
}

caData, err := ioutil.ReadFile(caFile)
if err != nil {
return nil, err
}

certPool := x509.NewCertPool()
certPool.AppendCertsFromPEM(caData)
return certPool, nil
}
Loading