[WIP] Add functionality to dynamically reload the VPA certs

vertical-pod-autoscaler/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/prometheus/common v0.4.1
 	github.com/prometheus/procfs v0.0.5 // indirect
 	github.com/stretchr/testify v1.4.0
+	github.com/fsnotify/fsnotify v1.4.9
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	k8s.io/api v0.18.3

vertical-pod-autoscaler/go.sum

@@ -35,6 +35,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -203,6 +205,7 @@ golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

vertical-pod-autoscaler/pkg/admission-controller/certs.go

@@ -17,11 +17,24 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
 	"io/ioutil"
+	"sync"
 
+	"github.com/fsnotify/fsnotify"
 	"k8s.io/klog"
 )
 
+// KeypairReloader structs holds cert path and certs
+type KeypairReloader struct {
+	certMu   sync.RWMutex
+	cert     *tls.Certificate
+	caCert   []byte
+	certPath string
+	keyPath  string
+	caPath   string
+}
+
 type certsContainer struct {
 	caCert, serverKey, serverCert []byte
 }
@@ -41,10 +54,75 @@ func readFile(filePath string) []byte {
 	return res
 }
 
-func initCerts(config certsConfig) certsContainer {
-	res := certsContainer{}
-	res.caCert = readFile(*config.clientCaFile)
-	res.serverCert = readFile(*config.tlsCertFile)
-	res.serverKey = readFile(*config.tlsPrivateKey)
-	return res
+// NewKeypairReloader will load certs on first run and trigger a goroutine for fsnotify watcher
+func NewKeypairReloader(config certsConfig) (*KeypairReloader, error) {
+	result := &KeypairReloader{
+		certPath: *config.tlsCertFile,
+		keyPath:  *config.tlsPrivateKey,
+		caCert:   readFile(*config.clientCaFile),
+	}
+	cert, err := tls.LoadX509KeyPair(*config.tlsCertFile, *config.tlsPrivateKey)
+	if err != nil {
+		return nil, err
+	}
+	result.cert = &cert
+
+	// creates a new file watcher
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			watcher.Close()
+		}
+	}()
+
+	if err := watcher.Add("/etc/tls-certs"); err != nil {
+		return nil, err
+	}
+
+	go func() {
+		for {
+			select {
+			// watch for events
+			case event := <-watcher.Events:
+				// fsnotify.create events will tell us if there are new certs
+				if event.Op&fsnotify.Create == fsnotify.Create {
+					klog.Info("Reloading certs")
+					if err := result.reload(); err != nil {
+						klog.Infof("Could not load new certs: %v", err)
+					}
+				}
+
+				// watch for errors
+			case err := <-watcher.Errors:
+				klog.Infof("error: %v", err)
+			}
+		}
+	}()
+
+	return result, nil
+}
+
+// reload loads updated cert and key whenever they are updated
+func (kpr *KeypairReloader) reload() error {
+	newCert, err := tls.LoadX509KeyPair(kpr.certPath, kpr.keyPath)
+	if err != nil {
+		return err
+	}
+	kpr.certMu.Lock()
+	defer kpr.certMu.Unlock()
+	kpr.cert = &newCert
+	return nil
+}
+
+// GetCertificateFunc will return function which will be used as tls.Config.GetCertificate
+func (kpr *KeypairReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		kpr.certMu.RLock()
+		defer kpr.certMu.RUnlock()
+		return kpr.cert, nil
+	}
 }

vertical-pod-autoscaler/pkg/admission-controller/config.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"context"
-	"crypto/tls"
 	"time"
 
 	admissionregistration "k8s.io/api/admissionregistration/v1"
@@ -45,16 +44,6 @@ func getClient() *kubernetes.Clientset {
 	return clientset
 }
 
-func configTLS(clientset *kubernetes.Clientset, serverCert, serverKey []byte) *tls.Config {
-	sCert, err := tls.X509KeyPair(serverCert, serverKey)
-	if err != nil {
-		klog.Fatal(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{sCert},
-	}
-}
-
 // register this webhook admission controller with the kube-apiserver
 // by creating MutatingWebhookConfiguration.
 func selfRegistration(clientset *kubernetes.Clientset, caCert []byte, namespace, serviceName, url string, registerByURL bool, timeoutSeconds int32) {

vertical-pod-autoscaler/pkg/admission-controller/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
 	"flag"
 	"fmt"
 
@@ -78,7 +79,11 @@ func main() {
 	metrics.Initialize(*address, healthCheck)
 	metrics_admission.Register()
 
-	certs := initCerts(*certsConfiguration)
+	// load certs
+	kpr, err := NewKeypairReloader(*certsConfiguration)
+	if err != nil {
+		klog.Fatal(err)
+	}
 
 	config, err := rest.InClusterConfig()
 	if err != nil {
@@ -128,13 +133,16 @@ func main() {
 	})
 	clientset := getClient()
 	server := &http.Server{
-		Addr:      fmt.Sprintf(":%d", *port),
-		TLSConfig: configTLS(clientset, certs.serverCert, certs.serverKey),
+		Addr: fmt.Sprintf(":%d", *port),
 	}
+	// this will check if there are new certs before every tls handshake
+	t := &tls.Config{GetCertificate: kpr.GetCertificateFunc()}
+	server.TLSConfig = t
+
 	url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort)
 	go func() {
 		if *registerWebhook {
-			selfRegistration(clientset, certs.caCert, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout))
+			selfRegistration(clientset, kpr.caCert, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout))
 		}
 		// Start status updates after the webhook is initialized.
 		statusUpdater.Run(stopCh)