Reload bootstrap kubeconfig if cert mgr failed to load valid certs

pkg/k8sclient/kubeconfig.go

@@ -22,10 +22,12 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"time"
 
 	certificatesv1 "k8s.io/api/certificates/v1"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -87,8 +89,28 @@ func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration tim
 	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
 	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
 		cfg := bootstrapKubeconfig
-		if current != nil {
-			cfg = config
+
+		// validate the kubeconfig
+		tempClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
+		} else {
+			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+			// tls unknown authority error is unrecoverable error with retry
+			if err != nil {
+				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
+					// reload and use bootstrapKubeconfig again
+					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					cfg = newBootstrapKubeconfig
+				} else {
+					logging.Errorf("failed to list pods with new certs: %v", err)
+				}
+			}
+
+			if current != nil {
+				cfg = config
+			}
 		}
 		return kubernetes.NewForConfig(cfg)
 	}