UPSTREAM: 00000: add dynamic certificate reloading

Origin-commit: 62a8e9edfee30a7c71787fcc8955981fed6fa5e1

cmd/kube-apiserver/app/aggregator.go

@@ -21,7 +21,6 @@ package app
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"strings"
 	"sync"
@@ -91,27 +90,14 @@ func createAggregatorConfig(
 		return nil, err
 	}
 
-	var err error
-	var certBytes, keyBytes []byte
-	if len(commandOptions.ProxyClientCertFile) > 0 && len(commandOptions.ProxyClientKeyFile) > 0 {
-		certBytes, err = ioutil.ReadFile(commandOptions.ProxyClientCertFile)
-		if err != nil {
-			return nil, err
-		}
-		keyBytes, err = ioutil.ReadFile(commandOptions.ProxyClientKeyFile)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	aggregatorConfig := &aggregatorapiserver.Config{
 		GenericConfig: &genericapiserver.RecommendedConfig{
 			Config:                genericConfig,
 			SharedInformerFactory: externalInformers,
 		},
 		ExtraConfig: aggregatorapiserver.ExtraConfig{
-			ProxyClientCert: certBytes,
-			ProxyClientKey:  keyBytes,
+			ProxyClientCert: commandOptions.ProxyClientCertFile,
+			ProxyClientKey:  commandOptions.ProxyClientKeyFile,
 			ServiceResolver: serviceResolver,
 			ProxyTransport:  proxyTransport,
 		},

cmd/kube-apiserver/app/server.go

@@ -448,7 +448,7 @@ func buildGenericConfig(
 	}
 	versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
 
-	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, versionedInformers)
+	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, genericConfig.Authentication.DynamicReloadFns, err = BuildAuthenticator(s, clientgoExternalClient, versionedInformers)
 	if err != nil {
 		lastErr = fmt.Errorf("invalid authentication config: %v", err)
 		return
@@ -512,7 +512,7 @@ func buildGenericConfig(
 }
 
 // BuildAuthenticator constructs the authenticator
-func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
+func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, map[string]genericapiserver.PostStartHookFunc, error) {
 	authenticatorConfig := s.Authentication.ToAuthenticationConfig()
 	if s.Authentication.ServiceAccounts.Lookup {
 		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(extclient)

cmd/kubelet/app/auth.go

@@ -76,7 +76,8 @@ func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletc
 		authenticatorConfig.TokenAccessReviewClient = client
 	}
 
-	authenticator, _, err := authenticatorConfig.New()
+	// ignore dynamic reload.  We may want this in the future
+	authenticator, _, _, err := authenticatorConfig.New()
 	return authenticator, err
 }
 

pkg/kubeapiserver/authenticator/config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package authenticator
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/go-openapi/spec"
@@ -33,11 +34,14 @@ import (
 	tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/certs"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
 	"k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
+
 	// Initialize all known client auth plugins.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	certutil "k8s.io/client-go/util/cert"
@@ -80,23 +84,28 @@ type Config struct {
 
 // New returns an authenticator.Request or an error that supports the standard
 // Kubernetes authentication mechanisms.
-func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
+func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, map[string]genericapiserver.PostStartHookFunc, error) {
 	var authenticators []authenticator.Request
 	var tokenAuthenticators []authenticator.Token
 	securityDefinitions := spec.SecurityDefinitions{}
+	dynamicReloadHooks := map[string]genericapiserver.PostStartHookFunc{}
 
 	// front-proxy, BasicAuth methods, local first, then remote
 	// Add the front proxy authenticator if requested
 	if config.RequestHeaderConfig != nil {
-		requestHeaderAuthenticator, err := headerrequest.NewSecure(
+		requestHeaderAuthenticator, dynamicReloadFn, err := headerrequest.NewSecure(
 			config.RequestHeaderConfig.ClientCA,
 			config.RequestHeaderConfig.AllowedClientNames,
 			config.RequestHeaderConfig.UsernameHeaders,
 			config.RequestHeaderConfig.GroupHeaders,
 			config.RequestHeaderConfig.ExtraHeaderPrefixes,
 		)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
+		}
+		dynamicReloadHooks["kube-apiserver-requestheader-reload"] = func(context genericapiserver.PostStartHookContext) error {
+			go dynamicReloadFn(context.StopCh)
+			return nil
 		}
 		authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, requestHeaderAuthenticator))
 	}
@@ -105,7 +114,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 	if len(config.BasicAuthFile) > 0 {
 		basicAuth, err := newAuthenticatorFromBasicAuthFile(config.BasicAuthFile)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, basicAuth))
 
@@ -119,32 +128,36 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 
 	// X509 methods
 	if len(config.ClientCAFile) > 0 {
-		certAuth, err := newAuthenticatorFromClientCAFile(config.ClientCAFile)
-		if err != nil {
-			return nil, nil, err
+		dynamicVerifier := certs.NewDynamicCA(config.ClientCAFile)
+		if err := dynamicVerifier.CheckCerts(); err != nil {
+			return nil, nil, nil, fmt.Errorf("unable to load client CA file %s: %v", config.ClientCAFile, err)
+		}
+		dynamicReloadHooks["kube-apiserver-clientCA-reload"] = func(context genericapiserver.PostStartHookContext) error {
+			go dynamicVerifier.Run(context.StopCh)
+			return nil
 		}
-		authenticators = append(authenticators, certAuth)
+		authenticators = append(authenticators, x509.NewDynamic(dynamicVerifier.GetVerifier, x509.CommonNameUserConversion))
 	}
 
 	// Bearer token methods, local first, then remote
 	if len(config.TokenAuthFile) > 0 {
 		tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
 	}
 	if len(config.ServiceAccountKeyFiles) > 0 {
 		serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
 	}
 	if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) && config.ServiceAccountIssuer != "" {
 		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuer, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
 	}
@@ -174,14 +187,14 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 			RequiredClaims:       config.OIDCRequiredClaims,
 		})
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, oidcAuth)
 	}
 	if len(config.WebhookTokenAuthnConfigFile) > 0 {
 		webhookTokenAuth, err := newWebhookTokenAuthenticator(config.WebhookTokenAuthnConfigFile, config.WebhookTokenAuthnCacheTTL, config.APIAudiences)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
 	}
@@ -206,9 +219,9 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 
 	if len(authenticators) == 0 {
 		if config.Anonymous {
-			return anonymous.NewAuthenticator(), &securityDefinitions, nil
+			return anonymous.NewAuthenticator(), &securityDefinitions, dynamicReloadHooks, nil
 		}
-		return nil, &securityDefinitions, nil
+		return nil, &securityDefinitions, dynamicReloadHooks, nil
 	}
 
 	authenticator := union.New(authenticators...)
@@ -221,7 +234,7 @@ func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, er
 		authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
 	}
 
-	return authenticator, &securityDefinitions, nil
+	return authenticator, &securityDefinitions, dynamicReloadHooks, nil
 }
 
 // IsValidServiceAccountKeyFile returns true if a valid public RSA key can be read from the given file

staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go

@@ -32,9 +32,9 @@ import (
 	"k8s.io/apiserver/pkg/authentication/request/websocket"
 	"k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authentication/token/cache"
+	"k8s.io/apiserver/pkg/server/certs"
 	webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
-	"k8s.io/client-go/util/cert"
 )
 
 // DelegatingAuthenticatorConfig is the minimal configuration needed to create an authenticator
@@ -56,41 +56,46 @@ type DelegatingAuthenticatorConfig struct {
 	RequestHeaderConfig *RequestHeaderConfig
 }
 
-func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
+type DynamicReloadFunc func(stopCh <-chan struct{})
+
+// New returns the authentication, the openapi info, dynamic reloading poststarthooks, or an error
+func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDefinitions, map[string]DynamicReloadFunc, error) {
 	authenticators := []authenticator.Request{}
 	securityDefinitions := spec.SecurityDefinitions{}
+	dynamicReloadHooks := map[string]DynamicReloadFunc{}
 
 	// front-proxy first, then remote
 	// Add the front proxy authenticator if requested
 	if c.RequestHeaderConfig != nil {
-		requestHeaderAuthenticator, err := headerrequest.NewSecure(
+		requestHeaderAuthenticator, dynamicReloadFn, err := headerrequest.NewSecure(
 			c.RequestHeaderConfig.ClientCA,
 			c.RequestHeaderConfig.AllowedClientNames,
 			c.RequestHeaderConfig.UsernameHeaders,
 			c.RequestHeaderConfig.GroupHeaders,
 			c.RequestHeaderConfig.ExtraHeaderPrefixes,
 		)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
+		dynamicReloadHooks["requestheader-reload"] = DynamicReloadFunc(dynamicReloadFn)
 		authenticators = append(authenticators, requestHeaderAuthenticator)
 	}
 
 	// x509 client cert auth
 	if len(c.ClientCAFile) > 0 {
-		clientCAs, err := cert.NewPool(c.ClientCAFile)
-		if err != nil {
-			return nil, nil, fmt.Errorf("unable to load client CA file %s: %v", c.ClientCAFile, err)
+		dynamicVerifier := certs.NewDynamicCA(c.ClientCAFile)
+		if err := dynamicVerifier.CheckCerts(); err != nil {
+			return nil, nil, nil, fmt.Errorf("unable to load client CA file %s: %v", c.ClientCAFile, err)
 		}
-		verifyOpts := x509.DefaultVerifyOptions()
-		verifyOpts.Roots = clientCAs
-		authenticators = append(authenticators, x509.New(verifyOpts, x509.CommonNameUserConversion))
+		dynamicReloadHooks["clientCA-reload"] = dynamicVerifier.Run
+
+		authenticators = append(authenticators, x509.NewDynamic(dynamicVerifier.GetVerifier, x509.CommonNameUserConversion))
 	}
 
 	if c.TokenAccessReviewClient != nil {
 		tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		cachingTokenAuth := cache.New(tokenAuth, false, c.CacheTTL, c.CacheTTL)
 		authenticators = append(authenticators, bearertoken.New(cachingTokenAuth), websocket.NewProtocolAuthenticator(cachingTokenAuth))
@@ -107,14 +112,14 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 
 	if len(authenticators) == 0 {
 		if c.Anonymous {
-			return anonymous.NewAuthenticator(), &securityDefinitions, nil
+			return anonymous.NewAuthenticator(), &securityDefinitions, dynamicReloadHooks, nil
 		}
-		return nil, nil, errors.New("No authentication method configured")
+		return nil, nil, nil, errors.New("No authentication method configured")
 	}
 
 	authenticator := group.NewAuthenticatedGroupAdder(unionauth.New(authenticators...))
 	if c.Anonymous {
 		authenticator = unionauth.NewFailOnError(authenticator, anonymous.NewAuthenticator())
 	}
-	return authenticator, &securityDefinitions, nil
+	return authenticator, &securityDefinitions, dynamicReloadHooks, nil
 }

staging/src/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader.go

@@ -17,9 +17,7 @@ limitations under the License.
 package headerrequest
 
 import (
-	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -28,7 +26,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	x509request "k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authentication/user"
-	utilcert "k8s.io/client-go/util/cert"
+	"k8s.io/apiserver/pkg/server/certs"
 )
 
 type requestHeaderAuthRequestHandler struct {
@@ -77,32 +75,25 @@ func trimHeaders(headerNames ...string) ([]string, error) {
 	return ret, nil
 }
 
-func NewSecure(clientCA string, proxyClientNames []string, nameHeaders []string, groupHeaders []string, extraHeaderPrefixes []string) (authenticator.Request, error) {
+type DynamicReloadFunc func(stopCh <-chan struct{})
+
+func NewSecure(clientCA string, proxyClientNames []string, nameHeaders []string, groupHeaders []string, extraHeaderPrefixes []string) (authenticator.Request, DynamicReloadFunc, error) {
 	headerAuthenticator, err := New(nameHeaders, groupHeaders, extraHeaderPrefixes)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if len(clientCA) == 0 {
-		return nil, fmt.Errorf("missing clientCA file")
+		return nil, nil, fmt.Errorf("missing clientCA file")
 	}
 
 	// Wrap with an x509 verifier
-	caData, err := ioutil.ReadFile(clientCA)
-	if err != nil {
-		return nil, fmt.Errorf("error reading %s: %v", clientCA, err)
-	}
-	opts := x509request.DefaultVerifyOptions()
-	opts.Roots = x509.NewCertPool()
-	certs, err := utilcert.ParseCertsPEM(caData)
-	if err != nil {
-		return nil, fmt.Errorf("error loading certs from  %s: %v", clientCA, err)
-	}
-	for _, cert := range certs {
-		opts.Roots.AddCert(cert)
+	dynamicVerifier := certs.NewDynamicCA(clientCA)
+	if err := dynamicVerifier.CheckCerts(); err != nil {
+		return nil, nil, fmt.Errorf("error reading %s: %v", clientCA, err)
 	}
 
-	return x509request.NewVerifier(opts, headerAuthenticator, sets.NewString(proxyClientNames...)), nil
+	return x509request.NewDynamicVerifier(dynamicVerifier.GetVerifier, headerAuthenticator, sets.NewString(proxyClientNames...)), dynamicVerifier.Run, nil
 }
 
 func (a *requestHeaderAuthRequestHandler) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {