Add ability to the VPA admission-controller to reload it's certificate v2

vertical-pod-autoscaler/go.mod

@@ -3,6 +3,7 @@ module k8s.io/autoscaler/vertical-pod-autoscaler
 go 1.21
 
 require (
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/golang/mock v1.6.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16

vertical-pod-autoscaler/go.sum

@@ -13,6 +13,8 @@ github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKf
 github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=

vertical-pod-autoscaler/pkg/admission-controller/certs.go

@@ -17,17 +17,18 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
 	"os"
+	"path"
+	"sync"
 
+	"github.com/fsnotify/fsnotify"
 	"k8s.io/klog/v2"
 )
 
-type certsContainer struct {
-	caCert, serverKey, serverCert []byte
-}
-
 type certsConfig struct {
 	clientCaFile, tlsCertFile, tlsPrivateKey *string
+	reload                                   *bool
 }
 
 func readFile(filePath string) []byte {
@@ -41,10 +42,59 @@ func readFile(filePath string) []byte {
 	return res
 }
 
-func initCerts(config certsConfig) certsContainer {
-	res := certsContainer{}
-	res.caCert = readFile(*config.clientCaFile)
-	res.serverCert = readFile(*config.tlsCertFile)
-	res.serverKey = readFile(*config.tlsPrivateKey)
-	return res
+type CertReloader struct {
+	tlsCertPath string
+	tlsKeyPath  string
+	cert        *tls.Certificate
+	mu          sync.RWMutex
+}
+
+func (cr *CertReloader) Start(stop <-chan struct{}) error {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+
+	if err = watcher.Add(path.Dir(cr.tlsCertPath)); err != nil {
+		return err
+	}
+	if err = watcher.Add(path.Dir(cr.tlsKeyPath)); err != nil {
+		return err
+	}
+	go func() {
+		defer watcher.Close()
+		for {
+			select {
+			case event := <-watcher.Events:
+				if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) {
+					klog.V(2).Info("New certificate found, reloading")
+					if err := cr.Load(); err != nil {
+						klog.Errorf("Failed to reload certificate: %s", err)
+					}
+				}
+			case err := <-watcher.Errors:
+				klog.Warningf("Error watching certificate files: %s", err)
+			case <-stop:
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+func (cr *CertReloader) Load() error {
+	cert, err := tls.LoadX509KeyPair(cr.tlsCertPath, cr.tlsKeyPath)
+	if err != nil {
+		return err
+	}
+	cr.mu.Lock()
+	defer cr.mu.Unlock()
+	cr.cert = &cert
+	return nil
+}
+
+func (cr *CertReloader) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cr.mu.RLock()
+	defer cr.mu.RUnlock()
+	return cr.cert, nil
 }

vertical-pod-autoscaler/pkg/admission-controller/config.go

@@ -33,14 +33,10 @@ const (
 	webhookConfigName = "vpa-webhook-config"
 )
 
-func configTLS(serverCert, serverKey []byte, minTlsVersion, ciphers string) *tls.Config {
+func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struct{}) *tls.Config {
 	var tlsVersion uint16
 	var ciphersuites []uint16
 	reverseCipherMap := make(map[string]uint16)
-	sCert, err := tls.X509KeyPair(serverCert, serverKey)
-	if err != nil {
-		klog.Fatal(err)
-	}
 
 	for _, c := range tls.CipherSuites() {
 		reverseCipherMap[c.Name] = c.ID
@@ -66,11 +62,30 @@ func configTLS(serverCert, serverKey []byte, minTlsVersion, ciphers string) *tls
 		klog.Fatal(fmt.Errorf("Unable to determine value for --min-tls-version (%s), must be either tls1_2 or tls1_3", minTlsVersion))
 	}
 
-	return &tls.Config{
+	config := &tls.Config{
 		MinVersion:   tlsVersion,
-		Certificates: []tls.Certificate{sCert},
 		CipherSuites: ciphersuites,
 	}
+	if *cfg.reload {
+		cr := CertReloader{
+			tlsCertPath: *cfg.tlsCertFile,
+			tlsKeyPath:  *cfg.tlsPrivateKey,
+		}
+		if err := cr.Load(); err != nil {
+			klog.Fatal(err)
+		}
+		if err := cr.Start(stop); err != nil {
+			klog.Fatal(err)
+		}
+		config.GetCertificate = cr.GetCertificate
+	} else {
+		cert, err := tls.LoadX509KeyPair(*cfg.tlsCertFile, *cfg.tlsPrivateKey)
+		if err != nil {
+			klog.Fatal(err)
+		}
+		config.Certificates = []tls.Certificate{cert}
+	}
+	return config
 }
 
 // register this webhook admission controller with the kube-apiserver

vertical-pod-autoscaler/pkg/admission-controller/main.go

@@ -58,6 +58,7 @@ var (
 		clientCaFile:  flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."),
 		tlsCertFile:   flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."),
 		tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."),
+		reload:        flag.Bool("reload-cert", false, "If set to true, reload leaf certificate."),
 	}
 	ciphers       = flag.String("tls-ciphers", "", "A comma-separated or colon-separated list of ciphers to accept.  Only works when min-tls-version is set to tls1_2.")
 	minTlsVersion = flag.String("min-tls-version", "tls1_2", "The minimum TLS version to accept.  Must be set to either tls1_2 (default) or tls1_3.")
@@ -86,7 +87,6 @@ func main() {
 	metrics.Initialize(*address, healthCheck)
 	metrics_admission.Register()
 
-	certs := initCerts(*certsConfiguration)
 	config := common.CreateKubeConfigOrDie(*kubeconfig, float32(*kubeApiQps), int(*kubeApiBurst))
 
 	vpaClient := vpa_clientset.NewForConfigOrDie(config)
@@ -133,12 +133,12 @@ func main() {
 	})
 	server := &http.Server{
 		Addr:      fmt.Sprintf(":%d", *port),
-		TLSConfig: configTLS(certs.serverCert, certs.serverKey, *minTlsVersion, *ciphers),
+		TLSConfig: configTLS(*certsConfiguration, *minTlsVersion, *ciphers, stopCh),
 	}
 	url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort)
 	go func() {
 		if *registerWebhook {
-			selfRegistration(kubeClient, certs.caCert, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout))
+			selfRegistration(kubeClient, readFile(*certsConfiguration.clientCaFile), namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout))
 		}
 		// Start status updates after the webhook is initialized.
 		statusUpdater.Run(stopCh)