Use TokenRequest API instead of calico-nodes service account token fo…

…r CNI kubeconfig.

With projected service account tokens in kubernetes, the service account tokens of pods
are bound to the lifetime of the corresponding pod. Therefore, it may lead to problems
if an external process re-uses the token of a pod.
The CNI binaries used the token of calico-node. However, in case calico-node got stopped
the corresponding token lost its validity and hence could no longer be used for CNI
operations. Usually, this automatically resolves over time, but there are some edge cases
where this is not possible, e.g. if calico-node is autoscaled in terms of resources and
the new resource requests would require preemption/eviction of an existing pod the CNI
operation to delete the network sandbox will fail due to the no longer valid token
(as calico-node was stopped beforehand).
This change switches over to using the TokenRequest API instead, i.e. creating new
tokens with limited validity. It would have been good to bind the token to an object,
e.g. to the corresponding node, but as of now only secret and pod are supported types
for binding tokens. Hence, the tokens are only limited in time and not bound to any
other kubernetes object.

app-policy/config/install/05-calico.yaml

@@ -96,6 +96,13 @@ rules:
       - get
       - list
       - watch
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    resourceNames:
+      - calico-node
+    verbs:
+      - create
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1beta1

calico/_includes/charts/calico/templates/calico-node-rbac.yaml

@@ -70,6 +70,14 @@ rules:
       - pods/status
     verbs:
       - patch
+  # Used for creating service account tokens to be used by the CNI plugin
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    resourceNames:
+      - calico-node
+    verbs:
+      - create
   # Calico monitors various CRDs for config.
   - apiGroups: ["crd.projectcalico.org"]
     resources:

calico/getting-started/kubernetes/hardway/install-node.md

@@ -121,6 +121,14 @@ rules:
       - pods/status
     verbs:
       - patch
+  # Used for creating service account tokens to be used by the CNI plugin
+  - apiGroups: [""]
+    resources:
+      - serviceaccounts/token
+    resourceNames:
+      - calico-node
+    verbs:
+      - create
   # Calico monitors various CRDs for config.
   - apiGroups: ["crd.projectcalico.org"]
     resources:

cni-plugin/Makefile

@@ -207,8 +207,8 @@ pkg/install/install.test: pkg/install/*.go
 
 .PHONY: test-install-cni
 ## Test the install
-test-install-cni: image pkg/install/install.test
-	cd pkg/install && CONTAINER_NAME=$(CNI_PLUGIN_IMAGE):latest-$(ARCH) ./install.test
+test-install-cni: run-k8s-apiserver image pkg/install/install.test
+	cd pkg/install && CONTAINER_NAME=$(CNI_PLUGIN_IMAGE):latest-$(ARCH) CERTS_PATH=$(CERTS_PATH) ./install.test
 
 ###############################################################################
 # CI/CD

cni-plugin/pkg/install/install.go

@@ -36,6 +36,7 @@ import (
 
 	"github.com/projectcalico/calico/libcalico-go/lib/logutils"
 	"github.com/projectcalico/calico/libcalico-go/lib/names"
+	"github.com/projectcalico/calico/node/pkg/cni"
 )
 
 type config struct {
@@ -476,7 +477,16 @@ contexts:
     user: calico
 current-context: calico-context`
 
-	data = strings.Replace(data, "TOKEN", kubecfg.BearerToken, 1)
+	clientset, err := cni.BuildClientSet()
+	if err != nil {
+		logrus.WithError(err).Fatal("Unable to create client for generating CNI token")
+	}
+	tr := cni.NewTokenRefresher(clientset, cni.NamespaceOfUsedServiceAccount(), cni.DefaultServiceAccountName)
+	tu, err := tr.UpdateToken()
+	if err != nil {
+		logrus.WithError(err).Fatal("Unable to create token for CNI kubeconfig")
+	}
+	data = strings.Replace(data, "TOKEN", tu.Token, 1)
 	data = strings.Replace(data, "__KUBERNETES_SERVICE_PROTOCOL__", getEnv("KUBERNETES_SERVICE_PROTOCOL", "https"), -1)
 	data = strings.Replace(data, "__KUBERNETES_SERVICE_HOST__", getEnv("KUBERNETES_SERVICE_HOST", ""), -1)
 	data = strings.Replace(data, "__KUBERNETES_SERVICE_PORT__", getEnv("KUBERNETES_SERVICE_PORT", ""), -1)

cni-plugin/pkg/install/install_test.go

@@ -2,6 +2,7 @@
 package install
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -13,6 +14,11 @@ import (
 	. "github.com/onsi/gomega"
 
 	log "github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 )
 
 var expectedDefaultConfig string = `{
@@ -79,13 +85,16 @@ func runCniContainer(tempDir string, binFolderWriteable bool, extraArgs ...strin
 	}
 	args := []string{
 		"run", "--rm", "--name", name,
+		"--net=host",
 		"-e", "SLEEP=false",
 		"-e", "KUBERNETES_SERVICE_HOST=127.0.0.1",
 		"-e", "KUBERNETES_SERVICE_PORT=6443",
 		"-e", "KUBERNETES_NODE_NAME=my-node",
+		"-e", "KUBECONFIG=/home/user/certs/kubeconfig",
 		"-v", tempDir + "/bin:" + binFolder,
 		"-v", tempDir + "/net.d:/host/etc/cni/net.d",
 		"-v", tempDir + "/serviceaccount:/var/run/secrets/kubernetes.io/serviceaccount",
+		"-v", os.Getenv("CERTS_PATH") + ":/home/user/certs",
 	}
 	args = append(args, extraArgs...)
 	image := os.Getenv("CONTAINER_NAME")
@@ -99,6 +108,54 @@ func runCniContainer(tempDir string, binFolderWriteable bool, extraArgs ...strin
 	return err
 }
 
+func createKubernetesClient() *kubernetes.Clientset {
+	certsPath := os.Getenv("CERTS_PATH")
+	if len(certsPath) == 0 {
+		Fail("CERTS_PATH env variable not set")
+	}
+	kubeconfigPath := certsPath + "/kubeconfig"
+	kubeconfigData, err := ioutil.ReadFile(kubeconfigPath)
+	if err != nil {
+		Fail(fmt.Sprintf("Failed to read kubeconfig file: %v", err))
+	}
+	// The client certificate/key do not necessarily reside in the location specified by kubeconfig => patch it directly
+	config, err := clientcmd.Load(kubeconfigData)
+	if err != nil {
+		Fail(fmt.Sprintf("Failed to load kubeconfig: %v", err))
+	}
+	certificate, err := ioutil.ReadFile(certsPath + "/admin.pem")
+	if err != nil {
+		Fail(fmt.Sprintf("Failed to read admin client certificate: %v", err))
+	}
+	key, err := ioutil.ReadFile(certsPath + "/admin-key.pem")
+	if err != nil {
+		Fail(fmt.Sprintf("Failed to read admin client key: %v", err))
+	}
+
+	overrides := &clientcmd.ConfigOverrides{
+		AuthInfo: api.AuthInfo{
+			ClientCertificate:     "",
+			ClientCertificateData: certificate,
+			ClientKey:             "",
+			ClientKeyData:         key,
+		},
+	}
+	adminAuthInfo := config.AuthInfos["admin"]
+	adminAuthInfo.ClientCertificate = ""
+	adminAuthInfo.ClientCertificateData = certificate
+	adminAuthInfo.ClientKey = ""
+	adminAuthInfo.ClientKeyData = key
+	kubeconfig, err := clientcmd.NewDefaultClientConfig(*config, overrides).ClientConfig()
+	if err != nil {
+		Fail(fmt.Sprintf("Failed to create kubeconfig: %v", err))
+	}
+	clientset, err := kubernetes.NewForConfig(kubeconfig)
+	if err != nil {
+		Fail(fmt.Sprintf("Could not create kubernetes client: %v", err))
+	}
+	return clientset
+}
+
 var _ = Describe("CNI installation tests", func() {
 	var tempDir string
 	BeforeEach(func() {
@@ -158,6 +215,29 @@ PuB/TL+u2y+iQUyXxLy3
 		if err != nil {
 			Fail(fmt.Sprintf("Failed to write k8s CA file for test: %v", err))
 		}
+
+		// Create namespace file for token refresh
+		k8sNamespace := []byte("kube-system")
+		var namespaceFile = fmt.Sprintf("%s/serviceaccount/namespace", tempDir)
+		err = ioutil.WriteFile(namespaceFile, k8sNamespace, 0755)
+		if err != nil {
+			Fail(fmt.Sprintf("Failed to write k8s namespace file: %v", err))
+		}
+
+		// Create calico-node service account
+		serviceAccount := &v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "calico-node",
+			},
+		}
+		_, err = createKubernetesClient().CoreV1().ServiceAccounts("kube-system").Create(context.Background(), serviceAccount, metav1.CreateOptions{})
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		// Cleanup calico-node service account
+		err := createKubernetesClient().CoreV1().ServiceAccounts("kube-system").Delete(context.Background(), "calico-node", metav1.DeleteOptions{})
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Context("Install with default values", func() {

node/Makefile

@@ -292,10 +292,10 @@ fv: run-k8s-apiserver
 	-v $(CERTS_PATH):/home/user/certs \
 	-e KUBECONFIG=/go/src/github.com/projectcalico/calico/hack/test/certs/kubeconfig \
 	-e ETCD_ENDPOINTS=http://$(LOCAL_IP_ENV):2379 \
-	$(CALICO_BUILD) ginkgo -cover -r -skipPackage vendor pkg/lifecycle/startup pkg/allocateip $(GINKGO_ARGS)
+	$(CALICO_BUILD) ginkgo -cover -r -skipPackage vendor pkg/lifecycle/startup pkg/allocateip pkg/cni $(GINKGO_ARGS)
 
 # Skip packages containing FV tests.
-UT_PACKAGES_TO_SKIP?=pkg/lifecycle/startup,pkg/allocateip,pkg/status
+UT_PACKAGES_TO_SKIP?=pkg/lifecycle/startup,pkg/allocateip,pkg/status,pkg/cni
 .PHONY: ut
 ut:
 	$(DOCKER_GO_BUILD) sh -c '$(GIT_CONFIG_SSH) ginkgo -r -skipPackage=$(UT_PACKAGES_TO_SKIP) $(GINKGO_ARGS)'